Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
This commit is contained in:
gti
@@ -0,0 +1,47 @@
|
||||
# Hub + local dev-only Firebird settings. Copy to .env locally; never commit
|
||||
# credentials or database URLs (this file is a template, .env is gitignored).
|
||||
#
|
||||
# For a real client Agent installation, use agent/.env.example instead — this
|
||||
# file's FIREBIRD_* values are for the disposable dev/test database only.
|
||||
|
||||
FIREBIRD_DATABASE_PATH=
|
||||
# Default MUST be 0. Only ever set to 1 against the authorized disposable
|
||||
# test database (see CLAUDE.md "Firebird write policy"); the agent fails
|
||||
# closed if the configured path is not exactly that file.
|
||||
FIREBIRD_WRITES=0
|
||||
|
||||
# --- Hub / PostgreSQL ---
|
||||
# DATABASE_URL takes precedence when set. If you override POSTGRES_DB/POSTGRES_USER
|
||||
# below, also set DATABASE_URL explicitly to match — the fallback URL built from
|
||||
# POSTGRES_PASSWORD alone assumes the gti/gti_fiscal defaults.
|
||||
DATABASE_URL=postgresql+psycopg://gti:development-only@localhost:5432/gti_fiscal
|
||||
POSTGRES_DB=gti_fiscal
|
||||
POSTGRES_USER=gti
|
||||
POSTGRES_PASSWORD=
|
||||
HUB_PORT=8000
|
||||
BASE_URL=http://localhost:8000
|
||||
APP_ENV=development
|
||||
LOG_LEVEL=INFO
|
||||
AGENT_ONLINE_THRESHOLD_SECONDS=120
|
||||
|
||||
# Local-dev-only agent bootstrap. Leave unset in any shared/pilot environment —
|
||||
# real Agents are created with central/scripts/register_agent.py.
|
||||
AGENT_DEV_TOKEN=
|
||||
|
||||
NCM_DATASET_URL=https://portalunico.siscomex.gov.br/classif/api/publico/nomenclatura/download/json
|
||||
CEST_DATASET_URL=https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18
|
||||
DATASET_STORAGE_PATH=./data/datasets
|
||||
HTTP_TIMEOUT_SECONDS=30
|
||||
|
||||
# Company context (Agent). Leave empty when C000004 has one row and CRT sources agree.
|
||||
# COMPANY_CODE= # C000004.CODIGO of the active company (when C000004 has several rows and EMISSOR cannot disambiguate)
|
||||
# COMPANY_CNPJ= # alternative selector (digits only); a stable, verifiable identifier
|
||||
# COMPANY_CRT_SOURCE= # EMISSOR | C000004 — required when the two CRT values disagree (emission programs read EMISSOR)
|
||||
|
||||
# Z.ai (OpenAI-compatible). NEVER commit the real key. Key lives only in .env
|
||||
AI_API_KEY=
|
||||
AI_BASE_URL=https://api.z.ai/api/paas/v4
|
||||
AI_MODEL=glm-5.3
|
||||
# Optional prices (USD per 1M tokens) - only set when confirmed with the provider
|
||||
AI_PRICE_IN_PER_M=
|
||||
AI_PRICE_OUT_PER_M=
|
||||
@@ -0,0 +1,6 @@
|
||||
# Shell scripts run on the Linux Hub host. With core.autocrlf=true on the
|
||||
# Windows release machine, `git archive`/checkout would otherwise ship them
|
||||
# with CRLF and `sh` rejects them ("set: Illegal option -").
|
||||
*.sh text eol=lf
|
||||
# systemd units are parsed on the same host; a trailing \r corrupts ExecStart.
|
||||
*.service text eol=lf
|
||||
@@ -0,0 +1,59 @@
|
||||
*.FDB
|
||||
*.FDB-journal
|
||||
*.FDB.lck
|
||||
.env
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
.superpowers/
|
||||
agent/*.sqlite
|
||||
agent/logs/
|
||||
agent/*.log
|
||||
|
||||
dist/
|
||||
build/
|
||||
|
||||
# --- release closure: secrets, runtime state, local configs, temporary artifacts ---
|
||||
.env.*
|
||||
!.env.example
|
||||
*.env
|
||||
!*.env.example
|
||||
*.secret
|
||||
secrets/
|
||||
credentials*
|
||||
*.pem
|
||||
*.key
|
||||
*.pfx
|
||||
*.log
|
||||
logs/
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.db
|
||||
*.dump
|
||||
*.zip
|
||||
Banco/
|
||||
bancos_teste/
|
||||
.playwright-mcp/
|
||||
.superpowers/
|
||||
deploy/hub/backups/
|
||||
deploy/hub/.env
|
||||
docs/runs/multi_db/_*
|
||||
scripts/_*
|
||||
.worktrees/
|
||||
*.tmp
|
||||
*.bak
|
||||
*.gz
|
||||
*.pkl
|
||||
.pytmp/
|
||||
sources_cache/legal/
|
||||
scratch_rtc_e2e/
|
||||
scratch_sandbox/
|
||||
# Missão 5: raw official dataset downloads for the Reference Data Registry stay off git -
|
||||
# only the manifest (URL, date, hash, size) is committed. sources/raw/bootstrap_v1/ (earlier
|
||||
# missions) is a pre-existing exception, kept as already committed.
|
||||
sources/raw/reference_registry/
|
||||
# Missão 9: per-installation Agent env files hold real AGENT_TOKEN secrets, plus local SQLite
|
||||
# state and logs - never commit any of it. Only the *_original.sha256.txt manifests under
|
||||
# work_copies/ (proof the real-client originals were never touched) are tracked.
|
||||
work_copies/agent_envs/
|
||||
work_copies/*.log
|
||||
scratch_enrich/
|
||||
@@ -0,0 +1,139 @@
|
||||
# Task 1 Report
|
||||
|
||||
## Status
|
||||
|
||||
Implemented the read-only discovery foundation. The FDB database was not opened,
|
||||
inspected, or modified.
|
||||
|
||||
## Files Changed
|
||||
|
||||
- `src/gti_discovery/__init__.py`: exports the read-only public interface.
|
||||
- `src/gti_discovery/read_only.py`: validates one `SELECT`/`WITH` statement and
|
||||
exposes `ReadOnlyConnection.execute` without connection write methods.
|
||||
- `tests/test_read_only.py`: covers accepted queries, normalization, prohibited
|
||||
statements, comments, multiple statements, quoted literals, and delegation.
|
||||
- `pyproject.toml`: defines the package and `src`-layout pytest configuration.
|
||||
- `.env.example`: documents an empty database-path placeholder without secrets.
|
||||
- `CLAUDE.md`: records the strict read-only project constraints and test command.
|
||||
|
||||
## Tests
|
||||
|
||||
Command:
|
||||
|
||||
```text
|
||||
python -m pytest tests/test_read_only.py -q
|
||||
```
|
||||
|
||||
Result: `18 passed in 0.04s`.
|
||||
|
||||
`git diff --check` also completed without whitespace errors.
|
||||
|
||||
## Concerns
|
||||
|
||||
- Commit was not created because Git has no configured author identity (`Author
|
||||
identity unknown`). Git configuration was not changed.
|
||||
- Pytest emits an existing `pytest-asyncio` deprecation warning because its event
|
||||
loop fixture scope is not configured; the focused tests still pass.
|
||||
- The validator intentionally rejects all SQL comments and any prohibited keyword
|
||||
outside quoted values. It does not attempt to parse the full Firebird grammar.
|
||||
- The wrapper expects the injected driver connection to provide `execute(sql,
|
||||
params)`; no Firebird driver is installed or required for this offline task.
|
||||
|
||||
## Reviewer Fixes
|
||||
|
||||
- Replaced the retained raw driver connection with a narrow execute-only
|
||||
delegate and added coverage proving `_connection`, `commit`, and `rollback`
|
||||
are unavailable on `ReadOnlyConnection`.
|
||||
- Added rejection for selectable procedure calls, `GEN_ID`, generator value
|
||||
expressions, `WITH LOCK`, and `FOR UPDATE`. The wrapper cannot enforce the
|
||||
driver's transaction/read-only mode, so deployment must use a read-only
|
||||
database account or driver-level setting.
|
||||
- Replaced global `split`/`join` normalization with whitespace collapsing only
|
||||
outside quoted literals; quoted contents are preserved byte-for-byte.
|
||||
- Added regression tests for side-effecting SELECT constructs and literal
|
||||
preservation.
|
||||
|
||||
## Fix Verification
|
||||
|
||||
Exact command:
|
||||
|
||||
```text
|
||||
python -m pytest tests -q
|
||||
```
|
||||
|
||||
Exact output:
|
||||
|
||||
```text
|
||||
C:\Users\Gustavo\AppData\Local\Programs\Python\Python312\Lib\site-packages\pytest_asyncio\plugin.py:299: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset.
|
||||
The event loop scope for asynchronous fixtures will default to the "fixture" caching scope. Future versions of pytest-asyncio will default the fixture scope to "function". Set the default fixture loop scope explicitly to avoid unexpected behavior.
|
||||
|
||||
warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET))
|
||||
....................... [100%]
|
||||
23 passed in 0.06s
|
||||
```
|
||||
|
||||
## Re-review Fixes
|
||||
|
||||
- Replaced the instance-bound driver delegate with an opaque-token lookup in a
|
||||
module-private registry. `ReadOnlyConnection` stores only the token; its
|
||||
bound `execute` method has the wrapper as `__self__` and no closure.
|
||||
- Added a regression test that checks `__self__`, `__closure__`, wrapper slots,
|
||||
and reachable slot values cannot expose the raw driver or `commit`.
|
||||
- Added conservative rejection of unknown and qualified function calls, while
|
||||
allowing a small deterministic-function allowlist. This covers selectable
|
||||
procedures, `GEN_ID`, UDF/external-function style calls, generator values,
|
||||
locking, and update forms. True read-only enforcement still requires a
|
||||
database read-only account or read-only transaction/driver configuration.
|
||||
|
||||
## Re-review Verification
|
||||
|
||||
Exact command:
|
||||
|
||||
```text
|
||||
python -m pytest tests -q
|
||||
```
|
||||
|
||||
Exact output:
|
||||
|
||||
```text
|
||||
C:\Users\Gustavo\AppData\Local\Programs\Python\Python312\Lib\site-packages\pytest_asyncio\plugin.py:299: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset.
|
||||
The event loop scope for asynchronous fixtures will default to the "fixture" caching scope. Future versions of pytest-asyncio will default the fixture scope to "function". Set the default fixture loop scope explicitly to avoid unexpected behavior.
|
||||
|
||||
warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET))
|
||||
.......................... [100%]
|
||||
26 passed in 0.07s
|
||||
```
|
||||
|
||||
## Third Fix Round
|
||||
|
||||
- Removed the module-level bound-method registry. `ReadOnlyConnection` now
|
||||
accepts only a narrow adapter-supplied execute callable and exposes no driver
|
||||
or connection attributes. Python process-level introspection is explicitly
|
||||
outside this guarantee.
|
||||
- Rejected all function-like SQL calls except a small deterministic-function
|
||||
allowlist, including qualified names with arbitrary whitespace around dots.
|
||||
Selectable procedures in `FROM`, generator calls, `GEN_ID`, `EXECUTE`, UDF,
|
||||
and external-function-style calls are rejected conservatively.
|
||||
- Documented that actual read-only enforcement requires a Firebird read-only
|
||||
account or read-only transaction/driver configuration.
|
||||
- Added regression coverage for spaced qualified calls and the practical public
|
||||
API boundary.
|
||||
|
||||
## Third Fix Verification
|
||||
|
||||
Exact command:
|
||||
|
||||
```text
|
||||
python -m pytest tests -q
|
||||
```
|
||||
|
||||
Exact output:
|
||||
|
||||
```text
|
||||
C:\Users\Gustavo\AppData\Local\Programs\Python\Python312\Lib\site-packages\pytest_asyncio\plugin.py:299: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset.
|
||||
The event loop scope for asynchronous fixtures will default to the "fixture" caching scope. Future versions of pytest-asyncio will default the fixture scope to "function". Set the default fixture loop scope explicitly to avoid unexpected behavior.
|
||||
|
||||
warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET))
|
||||
................................... [100%]
|
||||
35 passed in 0.09s
|
||||
```
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
# Final Fix Report — Control Panel Operational UX V1
|
||||
|
||||
Base: `c237820`
|
||||
|
||||
## Changes
|
||||
|
||||
- Legacy blockers without an exact installation binding remain fail-closed. Their detail page explains in Portuguese why resolution is unavailable and omits the resolution form. Direct HTML and JSON attempts return HTTP 409; missing blockers remain 404. Legacy details do not load context by guessing a company identity.
|
||||
- Resolution rejects derived `company_key` values longer than the database limit of 64 characters before creating context, job, or audit rows. No truncation or identity aliasing is performed.
|
||||
- Validation-error re-renders preserve the submitted key, value, scope, scope key, and reason, and reload current persisted context for a bound blocker.
|
||||
- Added regression coverage in `central/tests/test_blocker_resolution_panel.py` and `central/tests/test_control_panel_pages.py`.
|
||||
|
||||
## Verification
|
||||
|
||||
- TDD RED: focused tests initially failed on the missing company-key validation, unavailable-blocker response/UI, and form/context recovery behavior.
|
||||
- Focused: `python -m pytest central/tests/test_blocker_resolution_panel.py central/tests/test_control_panel_pages.py -q` — **26 passed**.
|
||||
- Full central suite: `python -m pytest central/tests -q` — **284 passed, 1 warning** (existing `openpyxl` print-area warning in `test_bootstrap_v1_extra.py`).
|
||||
- `git diff --check` — passed.
|
||||
|
||||
## Concerns
|
||||
|
||||
- Test output includes the existing `pytest-asyncio` configuration deprecation notice and one unrelated `openpyxl` warning; neither caused test failures.
|
||||
@@ -0,0 +1,101 @@
|
||||
# Task 1 Report: Presentation Helpers And Password Policy
|
||||
|
||||
## Status
|
||||
|
||||
Implemented Task 1 only. The existing server-rendered architecture and fiscal behavior were not changed.
|
||||
|
||||
## Changed Files
|
||||
|
||||
- `central/app/panel_presentation.py`: added timezone-aware UI datetime, age, duration, operational priority, and health-label helpers.
|
||||
- `central/app/config.py`: added `ui_timezone` (default `America/Bahia`) and `allow_weak_dev_password` settings.
|
||||
- `central/app/human_auth.py`: restored the 12-character minimum unless the process is non-production and `ALLOW_WEAK_DEV_PASSWORD=1`; the flag is read at call time.
|
||||
- `central/tests/test_panel_presentation.py`: added helper contract tests.
|
||||
- `central/tests/test_control_plane.py`: added explicit opt-in and production password-policy tests.
|
||||
- `docker-compose.yml`: explicitly enables weak development passwords for the local `fiscal-hub` service.
|
||||
|
||||
## RED Evidence
|
||||
|
||||
Command:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_panel_presentation.py central/tests/test_control_plane.py -q
|
||||
```
|
||||
|
||||
Result: collection failed with `ModuleNotFoundError: No module named 'central.app.panel_presentation'`, proving the new helper module was absent.
|
||||
|
||||
The control-plane tests were then run independently:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_control_plane.py -q
|
||||
```
|
||||
|
||||
Result: `2 failed, 3 passed`. Both new password-policy tests failed because the pre-change implementation accepted short passwords without the required environment gate.
|
||||
|
||||
## GREEN Evidence
|
||||
|
||||
Focused command:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_panel_presentation.py central/tests/test_control_plane.py -q
|
||||
```
|
||||
|
||||
Result: `10 passed`.
|
||||
|
||||
Focused command including configuration coverage:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_panel_presentation.py central/tests/test_control_plane.py central/tests/test_config.py -q
|
||||
```
|
||||
|
||||
Result: `11 passed`.
|
||||
|
||||
Full central regression:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests -q
|
||||
```
|
||||
|
||||
Result: `241 passed, 1 warning`. The warning is the existing `openpyxl` print-area warning from `central/tests/test_bootstrap_v1_extra.py`.
|
||||
|
||||
`git diff --check` and the staged diff check completed without whitespace errors.
|
||||
|
||||
## Commit
|
||||
|
||||
Implementation commit: `0846b9d1b076c60c57ad2a30a3071480a680bccf` (`feat: add operational panel presentation helpers`).
|
||||
|
||||
## Concerns
|
||||
|
||||
- The existing pytest environment emits one non-failing `pytest-asyncio` deprecation warning; the full central suite also emits the pre-existing `openpyxl` warning noted above.
|
||||
- `operational_priority` intentionally uses only operational blocker type, field, and reason signals; it does not infer fiscal severity.
|
||||
|
||||
## Reviewed Finding Fix
|
||||
|
||||
Finding: `create_admin` used the import-time `settings.allow_weak_dev_password` value as the default for the call-time environment lookup. If the flag was enabled during import and later removed, short passwords remained accepted.
|
||||
|
||||
Fix: `ALLOW_WEAK_DEV_PASSWORD` now defaults to `"0"` at every call. The existing `APP_ENV != production` check remains enforced, and Docker Compose still explicitly sets `ALLOW_WEAK_DEV_PASSWORD: "1"` for the local `fiscal-hub` service.
|
||||
|
||||
Regression RED command:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_control_plane.py::test_removed_weak_password_flag_is_fail_closed_after_import -q
|
||||
```
|
||||
|
||||
Result before the fix: `1 failed`; after reloading configuration/auth with the flag enabled, removing the flag did not raise `ValueError` for a short password.
|
||||
|
||||
Regression GREEN command:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_panel_presentation.py central/tests/test_control_plane.py -q
|
||||
```
|
||||
|
||||
Result: `11 passed`.
|
||||
|
||||
Full central regression:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests -q
|
||||
```
|
||||
|
||||
Result: `242 passed, 1 warning`; the warning is the existing `openpyxl` print-area warning.
|
||||
|
||||
Fix commit: `de571e3ac2ce214677e1f9baa673181350b7d411` (`fix: fail closed weak password opt-in`).
|
||||
@@ -0,0 +1,193 @@
|
||||
# Task 2 Report: Bounded Operational Query View-Models
|
||||
|
||||
## Status
|
||||
|
||||
Implemented Task 2 only. The control-plane query layer now supplies bounded operational view-models while preserving the existing JSON keys and technical health derivation. No fiscal logic, database schema, Firebird access, or remote write capability was changed.
|
||||
|
||||
## RED Evidence
|
||||
|
||||
Added `central/tests/test_operational_view_models.py` before changing production code.
|
||||
|
||||
Command:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_operational_view_models.py -q
|
||||
```
|
||||
|
||||
Result before implementation: `6 failed`.
|
||||
|
||||
The failures were the expected missing-contract failures: `dashboard_summary` did not accept `now`, `list_installations` did not accept `sort`, `list_jobs` did not accept `client`, `list_blockers` did not accept `client`, and `blocker_detail` did not exist. This confirmed the tests exercised the new behavior rather than existing behavior.
|
||||
|
||||
## GREEN Evidence
|
||||
|
||||
Focused Task 2 tests:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_operational_view_models.py -q
|
||||
```
|
||||
|
||||
Result: `6 passed`.
|
||||
|
||||
Focused compatibility tests:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_operational_view_models.py central/tests/test_control_plane.py central/tests/test_jobs.py -q
|
||||
```
|
||||
|
||||
Result: `21 passed`.
|
||||
|
||||
Central regression suite:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests -q
|
||||
```
|
||||
|
||||
Result: `248 passed, 1 warning`.
|
||||
|
||||
The warning is the pre-existing `openpyxl` print-area warning from `central/tests/test_bootstrap_v1_extra.py`. The pytest environment also emits the existing `pytest-asyncio` fixture-loop-scope deprecation warning.
|
||||
|
||||
Additional verification:
|
||||
|
||||
```text
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Result: no whitespace errors. Git reported only its existing LF/CRLF normalization notice for the edited Python files.
|
||||
|
||||
## Changed Files
|
||||
|
||||
- `central/app/control_plane.py`: added bounded dashboard aggregates and attention items; enriched installation, job, and blocker view-models; added filters, stable pagination, worst-coverage sorting, presentation fields, and bounded blocker detail relations.
|
||||
- `central/tests/test_operational_view_models.py`: added focused TDD coverage for health reconciliation, distinct safe-blocked products, attention ordering/cap, fiscal/technical separation, job identity/duration, blocker filters/priorities, and detail relations.
|
||||
|
||||
## Contract Notes
|
||||
|
||||
- Dashboard health counts preserve lowercase compatibility keys and add uppercase `health_counts`.
|
||||
- `safe_blocked_products` counts distinct active `(tenant_id, product_code)` pairs; `open_blockers` counts active rows.
|
||||
- Safe fiscal blockers do not alter `technical_health`.
|
||||
- API timestamps remain ISO UTC values; presentation fields use Task 1 helpers.
|
||||
- Collection queries use joins/aggregates. Blocker detail related events, jobs, and audit rows are bounded and bulk-loaded.
|
||||
- Existing `resolved` behavior remains supported while `status` adds explicit `OPEN`, `RESOLVED`, and `ALL` handling.
|
||||
|
||||
## Commit
|
||||
|
||||
Implementation commit: `d729710` (`feat: add bounded operational view models`).
|
||||
|
||||
## Concerns
|
||||
|
||||
- `BlockedProduct` has no persisted installation foreign key. Blocker list/detail identity therefore uses the lowest-ID installation for the tenant as the representative context; agent and installation filters check tenant installation membership without duplicating blocker rows.
|
||||
- The existing non-failing pytest environment warnings remain as noted above.
|
||||
|
||||
## Task 2 Re-Review Fix Report
|
||||
|
||||
### Findings Fixed
|
||||
|
||||
- Unified fiscal completion rounding with deterministic one-decimal `ROUND_HALF_UP` behavior in Python and SQL. The SQL ratio is explicitly numeric before two-argument `ROUND`, preserving SQLite/PostgreSQL compatibility.
|
||||
- Extended `blocker_fingerprint` with an optional installation identifier. Legacy calls without an installation retain the original hash; authenticated upserts scope both generated and supplied fingerprints to the exact installation, so identical reports from two installations remain distinct.
|
||||
- Normalized blocker attention candidate fields with SQL `UPPER`, matching the normalized breakdown behavior for lowercase `cfop`, `st`, and other required fields.
|
||||
|
||||
### Re-Review RED Evidence
|
||||
|
||||
Command after adding the three regression tests:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_operational_view_models.py::test_worst_coverage_sort_cannot_disagree_with_one_decimal_display central/tests/test_operational_view_models.py::test_lowercase_required_blocker_field_reaches_attention_candidates central/tests/test_blocked_products.py::test_authenticated_installations_get_distinct_fingerprints_for_identical_blockers -q
|
||||
```
|
||||
|
||||
Result before implementation: `3 failed`.
|
||||
|
||||
The failures reproduced the SQL/Python 6.3 versus 6.2 ordering mismatch, omitted lowercase blocker attention, and deduplication of identical blocker reports across installations.
|
||||
|
||||
### Re-Review GREEN Evidence
|
||||
|
||||
Focused Task 2/control/blocker tests:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_operational_view_models.py central/tests/test_control_plane.py central/tests/test_jobs.py central/tests/test_blocked_products.py central/tests/test_operational_state.py -q
|
||||
```
|
||||
|
||||
Result: `37 passed`.
|
||||
|
||||
Full central suite:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests -q
|
||||
```
|
||||
|
||||
Result: `256 passed, 1 warning`.
|
||||
|
||||
The warning remains the pre-existing `openpyxl` print-area warning; the environment also emits the existing `pytest-asyncio` fixture-loop-scope deprecation warning.
|
||||
|
||||
PostgreSQL SQL compatibility check:
|
||||
|
||||
```text
|
||||
python -c "from sqlalchemy import select; from sqlalchemy.dialects.postgresql import dialect; from central.app.control_plane import _fiscal_completion_expression; print(select(_fiscal_completion_expression()).compile(dialect=dialect()))"
|
||||
```
|
||||
|
||||
Result: the expression compiles as numeric SQL using two-argument `ROUND`.
|
||||
|
||||
### Re-Review Fix Commit
|
||||
|
||||
Implementation commit: `2ee271c` (`fix: close remaining task 2 review findings`).
|
||||
|
||||
### Re-Review Concerns
|
||||
|
||||
- Legacy no-installation payloads intentionally retain their original fingerprint and tenant-level behavior; authenticated installation-scoped payloads use the new namespace.
|
||||
- The existing non-failing pytest environment warnings remain as noted above.
|
||||
|
||||
## Task 2 Review Fix Report
|
||||
|
||||
### Findings Fixed
|
||||
|
||||
- Added nullable `blocked_products.installation_id` with a foreign key to `installations.id` and index in Alembic revision `0018_blocker_installation_identity`; updated the SQLAlchemy model.
|
||||
- Updated blocker upsert and authenticated job completion to persist the authenticated Agent installation. Agent-facing blocker list/reprocess/export paths and control-panel agent detail routes now use exact installation ownership. Legacy null-owned rows remain tenant-level and render without fabricated server context.
|
||||
- Replaced dashboard installation/blocker materialization with SQL health counts, snapshot totals, open-row count, distinct tenant/product count, and blocker-field breakdown. Health, blocker, and version attention candidates are individually bounded before Python merge/truncation.
|
||||
- Replaced Python health filtering/slicing with a portable SQL `CASE` expression using the supplied `now`, failure/job fields, heartbeat cutoff, SQL count, order, offset, and limit.
|
||||
- Worst-coverage ordering now uses the same derived complete/total percentage, clamped and rounded consistently with presentation.
|
||||
|
||||
### Review-Fix RED Evidence
|
||||
|
||||
Command after adding the review regressions:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_operational_view_models.py central/tests/test_jobs.py -q
|
||||
```
|
||||
|
||||
Result: the new regressions failed as expected: exact ownership was unsupported, legacy blockers received the fabricated representative installation, dashboard SQL `DISTINCT` aggregation was absent, health-filtered SQL pagination was absent, and stored coverage ordering disagreed with derived coverage. The API assertion initially also exposed a test setup omission; the test was corrected to heartbeat/register the installation before implementation.
|
||||
|
||||
### Review-Fix GREEN Evidence
|
||||
|
||||
Focused review and compatibility tests:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_operational_view_models.py central/tests/test_control_plane.py central/tests/test_jobs.py central/tests/test_blocked_products.py -q
|
||||
```
|
||||
|
||||
Result: `30 passed`.
|
||||
|
||||
Full central suite:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests -q
|
||||
```
|
||||
|
||||
Result: `253 passed, 1 warning`.
|
||||
|
||||
The warning remains the pre-existing `openpyxl` print-area warning. The test environment also emits the existing `pytest-asyncio` fixture-loop-scope deprecation warning.
|
||||
|
||||
Additional verification:
|
||||
|
||||
```text
|
||||
python -m compileall -q central/app central/scripts
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Both completed without errors; Git's LF/CRLF normalization notices are environmental only.
|
||||
|
||||
### Review-Fix Commit
|
||||
|
||||
Implementation commit: `e35417f` (`fix: close task 2 operational review findings`).
|
||||
|
||||
### Review-Fix Concerns
|
||||
|
||||
- Legacy blocker rows intentionally remain tenant-level because their installation ownership is unavailable; they render `-`/null server context and are never assigned a representative installation.
|
||||
- The existing non-failing pytest environment warnings remain as noted above.
|
||||
@@ -0,0 +1,172 @@
|
||||
# Task 3 Report: Safe Blocker Resolution Workflow
|
||||
|
||||
## Status
|
||||
|
||||
Implemented Task 3 only. The old hidden-context blocker action was replaced by a structured, server-rendered, CSRF-protected resolution workflow. The JSON control-panel route now uses the same validated transaction service and rejects arbitrary context text.
|
||||
|
||||
## RED
|
||||
|
||||
Created `central/tests/test_blocker_resolution_panel.py` before production implementation.
|
||||
|
||||
Initial focused command:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_blocker_resolution_panel.py -q
|
||||
```
|
||||
|
||||
Expected RED result:
|
||||
|
||||
```text
|
||||
ModuleNotFoundError: No module named 'central.app.blocker_resolution'
|
||||
```
|
||||
|
||||
The failure occurred during collection because the required production module did not yet exist.
|
||||
|
||||
## Implementation
|
||||
|
||||
- Added `central/app/blocker_resolution.py` with structured input validation through `validate_entry`, workflow-scope restrictions, exact PRODUCT scope-key enforcement, persisted installation/company context derivation, context upsert, bounded `REPROCESS_PRODUCTS` job creation, sanitized audit recording, and one commit with rollback on failure.
|
||||
- Added `GET /blockers/{blocker_id}` with installation-aware blocker detail and context form.
|
||||
- Added `POST /blockers/{blocker_id}/resolve-context` with human authentication and CSRF enforcement.
|
||||
- Updated `/v1/blockers/{blocker_id}/resolve-human` and added `/v1/blockers/{blocker_id}/resolve-context` to call the same service; neither accepts arbitrary context text.
|
||||
- Removed the blocker-list hidden fixed-context form and `data-confirm` action.
|
||||
- Added focused coverage for enum validation, unknown keys, unsupported NCM scope, empty reason, missing PRODUCT scope key, authentication, CSRF, context/job/audit persistence, exact product scoping, and secret redaction.
|
||||
|
||||
## GREEN
|
||||
|
||||
Focused command:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_blocker_resolution_panel.py -q
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
5 passed, 1 warning
|
||||
```
|
||||
|
||||
Central suite:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests -q
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
261 passed, 1 warning
|
||||
```
|
||||
|
||||
The warnings are existing third-party test-environment warnings from `pytest_asyncio` configuration and `openpyxl` print-area parsing.
|
||||
|
||||
## Safety Review
|
||||
|
||||
- No Firebird, AI, SQL execution, shell, credential, or remote-write path was added.
|
||||
- The workflow accepts only `scope`, `scope_key`, `key`, `value`, and `reason`; context keys and enum values are validated against existing shared rules.
|
||||
- Company context is derived from persisted blocker/installation data, not arbitrary client `company_key` input.
|
||||
- Jobs contain exactly the blocked product code.
|
||||
- Audit context passes through existing redaction before persistence.
|
||||
|
||||
## Concerns
|
||||
|
||||
- Live PostgreSQL and agent execution were not exercised; verification used the repository's offline SQLite test fixtures.
|
||||
- The blocker remains open until the agent's normal reprocess completion path reports resolution; this prevents marking it resolved before the bounded job succeeds.
|
||||
|
||||
## Review Fixes
|
||||
|
||||
### RED
|
||||
|
||||
Added regression tests before changing production code for the four review findings:
|
||||
|
||||
- Legacy blockers with `installation_id=NULL` must fail closed without selecting an Agent or creating work.
|
||||
- Cross-tenant installation and Agent pointers must fail before context, job, or audit mutation.
|
||||
- Oversized values/reasons and non-string fields must be rejected cleanly.
|
||||
- Successful web resolution must redirect to the blocker detail page.
|
||||
|
||||
Focused command before the fixes:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_blocker_resolution_panel.py -q
|
||||
```
|
||||
|
||||
RED result:
|
||||
|
||||
```text
|
||||
4 failed, 5 passed
|
||||
```
|
||||
|
||||
The failures reproduced the legacy Agent fallback, cross-tenant acceptance, unbounded/malformed input, and `/blockers` redirect behavior.
|
||||
|
||||
### GREEN
|
||||
|
||||
- Resolution now requires a non-null exact installation and verifies both installation and Agent tenant ownership before mutation.
|
||||
- `value` is required to be a string of at most 512 characters; `reason` is required to be a string of at most 2000 characters.
|
||||
- Boundary-sized audit data retains all required audit fields without whole-context truncation.
|
||||
- Web success redirects to `/blockers/{blocker_id}`.
|
||||
|
||||
Focused command:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_blocker_resolution_panel.py -q
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
9 passed, 1 warning
|
||||
```
|
||||
|
||||
Full central command:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests -q
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
265 passed, 1 warning
|
||||
```
|
||||
|
||||
The existing `pytest_asyncio` configuration and `openpyxl` print-area warnings remain unchanged.
|
||||
|
||||
## UTF-8 Byte-Length Re-Review Fix
|
||||
|
||||
### RED
|
||||
|
||||
Added `test_resolution_input_rejects_multibyte_utf8_at_ascii_character_limits` before changing production code. It submits `é` repeated 512 times as `value` and 2000 times as `reason`; both are within the existing character counts but exceed the corresponding UTF-8 byte budgets.
|
||||
|
||||
Focused command:
|
||||
|
||||
```text
|
||||
python -m pytest central/tests/test_blocker_resolution_panel.py -q
|
||||
```
|
||||
|
||||
RED result:
|
||||
|
||||
```text
|
||||
1 failed, 9 passed
|
||||
```
|
||||
|
||||
The failure confirmed character-count validation accepted the multibyte inputs.
|
||||
|
||||
### GREEN
|
||||
|
||||
- `value` and `reason` limits now use `len(value.encode("utf-8"))` and `len(reason.encode("utf-8"))`.
|
||||
- Existing ASCII boundary inputs remain accepted.
|
||||
- Multibyte inputs fail closed with explicit UTF-8 byte-limit validation errors, before persistence or audit sanitization.
|
||||
- Redaction and remote-write boundaries were not changed.
|
||||
|
||||
Focused result:
|
||||
|
||||
```text
|
||||
10 passed, 1 warning
|
||||
```
|
||||
|
||||
Full central result:
|
||||
|
||||
```text
|
||||
266 passed, 1 warning
|
||||
```
|
||||
|
||||
The existing `pytest_asyncio` configuration and `openpyxl` print-area warnings remain unchanged.
|
||||
@@ -0,0 +1,44 @@
|
||||
# Task 4 Report: Server-Rendered Operational Pages
|
||||
|
||||
STATUS: COMPLETE
|
||||
|
||||
## Implemented
|
||||
|
||||
- Added authenticated page tests covering dashboard, agents, jobs, blockers, agent detail, blocker detail, filters, pagination state, human dates, and safe-form constraints.
|
||||
- Added server-side route filters for agents, blockers, jobs, and agent events while preserving bounded view-model calls.
|
||||
- Reworked the Jinja pages with Portuguese operational labels, separate technical/fiscal statuses, empty states, detail links, accessible controls, semantic table captions/headers, and secondary technical IDs.
|
||||
- Added dashboard attention, failed-job, outdated-agent, and blocker summary sections.
|
||||
- Preserved the Task 3 blocker detail and CSRF-protected resolution form, with evidence and related operational records.
|
||||
- Replaced raw primary timestamp rendering with humanized presentation values.
|
||||
- Replaced the old confirmation JavaScript with harmless progressive enhancement; no `window.confirm` or `data-confirm` remains in the panel pages.
|
||||
- Added responsive table overflow, mobile breakpoints, visible focus states, and text-bearing status badges.
|
||||
|
||||
## Verification
|
||||
|
||||
- RED: `python -m pytest central/tests/test_control_panel_pages.py -q` failed 5 tests against the pre-change templates/routes.
|
||||
- Focused: `python -m pytest central/tests/test_control_panel_pages.py -q` passed 5 tests.
|
||||
- Central suite: `python -m pytest central/tests -q` passed 271 tests with 1 existing openpyxl warning.
|
||||
- `git diff --check` passed.
|
||||
- No credentials, tokens, cookies, or secrets were added.
|
||||
|
||||
## Concerns
|
||||
|
||||
- The blocker page's affected-client count is derived from the bounded filtered result page; the canonical global blocker/product metrics continue to come from `dashboard_summary`.
|
||||
- Browser viewport screenshots were not run in this offline task; responsive behavior is covered by CSS breakpoints and horizontal table scrolling.
|
||||
|
||||
Commit message: `feat: refine operational control panel pages`
|
||||
|
||||
## Task 4 review fixes
|
||||
|
||||
- Added a dashboard list of at most 10 recent failed jobs with client, installation display name, hostname, job ID/kind, short error, created/finished display and age, and a job-ID-filtered link. The failed-job total and empty state remain visible.
|
||||
- Replaced page-local/global blocker metric mixing with SQL-backed aggregates over the complete relation selected by the current filters. Counts use distinct `(tenant_id, product_code)` identity and distinct tenant IDs; OPEN, RESOLVED, and ALL behavior is covered. Breakdown now follows the filtered relation.
|
||||
- Agent detail now loads its installation, agent, tenant, and snapshot by agent ID, without scanning the first 100 installations. Agent blocker breakdown and its filtered link include only blockers whose installation belongs to that agent; tenant-only legacy records are excluded.
|
||||
- Added event since/until filters and page-size+1 lookahead, preserving filters in pagination links and omitting Next on the final page. Job references link to jobs filtered by job ID; product references link to blockers filtered by agent/product.
|
||||
- Reprocess-all now opens an explicit server-rendered confirmation page with a required confirmation checkbox, CSRF token, and cancel link. Missing confirmation is rejected; valid confirmation retains the guarded enqueue path.
|
||||
- Added route regressions for these cases, including 55+ blockers, repeated product code across tenants, resolved/all status, agent ID beyond 100, exact blocker attribution, date filtering/pagination, links, and confirmation/CSRF behavior.
|
||||
- RED evidence: the new dashboard-row and date-filter cases failed before implementation because those views were absent; after correcting test setup, the new assertions detected the unimplemented behaviors. Focused regressions are green after implementation.
|
||||
- Verification: `python -m pytest central/tests/test_control_panel_pages.py -q` — 9 passed; `python -m pytest central/tests -q` — 275 passed, 1 existing openpyxl warning; `git diff --check` — passed.
|
||||
- Route smoke coverage is provided by authenticated TestClient requests. CSS breakpoint review confirmed the existing 1100px, 768px, and 520px responsive rules and table overflow remain intact.
|
||||
- The earlier concern that blocker metrics were page-local is resolved by the complete filtered SQL aggregates above. No writes to Firebird databases were performed.
|
||||
|
||||
Commit message: `fix: close operational panel page review findings`
|
||||
@@ -0,0 +1,79 @@
|
||||
# Task 5 Report: Fixture Coverage, Documentation, And Regression Tests
|
||||
|
||||
## Result
|
||||
|
||||
Completed the fixture regression coverage and operational UX documentation.
|
||||
The fixture retains its explicit non-production command gate and the default
|
||||
100-Agent/500-event/100-job shape. Its deterministic blocker count is now 51:
|
||||
50 distinct open tenant/product pairs, including two open blocker rows for the
|
||||
same first product. Safe-blocked snapshot counts match the fixture's open
|
||||
product pairs. Failed jobs have duration timestamps, and linked events point to
|
||||
their job and, where present, the correct blocker/product and installation.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
1. Added assertions for the additional same-product blocker, unique fingerprints,
|
||||
exact UUID5 installation identities, ownership, failed-job durations, and
|
||||
job/blocker/product event links. The first focused run was RED as expected:
|
||||
fixture returned 50 blockers while the requirement expected 51.
|
||||
2. After implementing the blocker/job/event fixtures, added an assertion that
|
||||
each snapshot's `safe_blocked` value matched its open product pairs. The
|
||||
focused run was RED as expected because the old sample formula reported
|
||||
blocked products for installations without an open blocker.
|
||||
3. Updated the deterministic snapshot values to match the generated open
|
||||
product pairs. The focused test then passed.
|
||||
|
||||
## Verification commands
|
||||
|
||||
- RED: `python -m pytest central/tests/test_control_panel_fixture.py -q` —
|
||||
failed on the expected blocker count gap (`50 != 51`).
|
||||
- RED: `python -m pytest central/tests/test_control_panel_fixture.py -q` —
|
||||
failed on the expected safe-blocked snapshot mismatch (`1 != 0`).
|
||||
- GREEN: `python -m pytest central/tests/test_control_panel_fixture.py -q` —
|
||||
passed (`1 passed`).
|
||||
- Full central suite: `python -m pytest central/tests -q` — passed
|
||||
(`275 passed`). Pytest emitted its existing `pytest-asyncio` fixture-loop
|
||||
deprecation warning and one existing `openpyxl` print-area warning.
|
||||
- `git diff --cached --check` — passed with no whitespace errors.
|
||||
|
||||
## Documentation review
|
||||
|
||||
Updated `docs/CONTROL_PANEL_V1.md` to link to the new operational UX contract
|
||||
and distinguish the installation-wide confirmation step from the validated
|
||||
blocker context workflow. Added `docs/CONTROL_PANEL_OPERATIONAL_UX_V1.md`
|
||||
covering the health invariant, technical/fiscal distinction, blocker/product
|
||||
counts, dashboard attention and empty states, identity-first views, timestamp
|
||||
and timezone presentation, blocker filters and summaries, operational-only
|
||||
priority tiers, safe resolution/auth/CSRF/audit fields, reprocess confirmation,
|
||||
password policy, performance limits, responsiveness, fixture command, and
|
||||
write/deployment boundaries.
|
||||
|
||||
## Commit
|
||||
|
||||
- Implementation and product documentation: `4ba0fa8`
|
||||
(`test: complete control panel fixture UX coverage`)
|
||||
|
||||
## Concerns
|
||||
|
||||
- Resolved in the timestamp tooltip follow-up below: templates now use raw ISO
|
||||
UTC timestamps in `title` and retain localized human-readable visible text.
|
||||
- Full central tests retain the two pre-existing dependency warnings described
|
||||
above; they did not fail the suite.
|
||||
|
||||
## Timestamp tooltip follow-up
|
||||
|
||||
Resolved the full technical timestamp concern: agent, job, blocker, and
|
||||
populated dashboard failure timestamps now keep their localized human-readable
|
||||
text visible and expose the complete ISO UTC timestamp (including seconds and
|
||||
`+00:00`) in the `title` attribute. Blocker detail includes first- and
|
||||
last-seen timestamps. The existing raw ISO serializer now explicitly normalizes
|
||||
naive SQLite timestamps as UTC, matching the presentation layer's existing
|
||||
interpretation without changing the view-model shape.
|
||||
|
||||
### TDD and verification
|
||||
|
||||
- RED: `python -m pytest central/tests/test_control_panel_pages.py -q -k "localized_last_seen or localized_created or localized_seen_times or bounded_failed_job_rows"` — 4 expected failures because the tooltips did not contain the required raw ISO values.
|
||||
- GREEN: `python -m pytest central/tests/test_control_panel_pages.py -q` — passed (`12 passed`).
|
||||
- Full suite: `python -m pytest central/tests -q` — passed (`278 passed`, one existing `openpyxl` warning; pytest-asyncio also emits its existing fixture-loop deprecation warning).
|
||||
- `git diff --check` — passed.
|
||||
- Commit message: `fix: show technical timestamps in tooltips`.
|
||||
@@ -0,0 +1,130 @@
|
||||
# GTI Discovery
|
||||
|
||||
**Primeira leitura obrigatória, antes de qualquer missão fiscal nova**: `docs/PROJECT_STATE.md`
|
||||
(versões Hub/Agent, migration head de cada branch, status do Reference Data Registry, decisões
|
||||
ADMIN pendentes, o que já foi feito e o que falta, riscos abertos). Mantê-lo atualizado ao final
|
||||
de cada missão que mude esse estado.
|
||||
|
||||
This project is strictly read-only. Discovery code must use a database-level
|
||||
read-only account or transaction/driver mode; lexical SQL validation is only
|
||||
defense-in-depth. Discovery code must not perform DML or DDL,
|
||||
modify database objects or permissions, or store credentials and other secrets.
|
||||
|
||||
Run the offline tests with:
|
||||
|
||||
```text
|
||||
# Suíte completa canônica (1160+ testes: raiz + Hub + Agent):
|
||||
python -m pytest tests central/tests agent/tests -q
|
||||
|
||||
# Apenas testes unitários offline da raiz:
|
||||
python -m pytest tests -q
|
||||
```
|
||||
|
||||
|
||||
## Firebird write policy
|
||||
|
||||
Default policy:
|
||||
- All client/production Firebird databases are STRICTLY READ-ONLY.
|
||||
- Never perform INSERT, UPDATE, DELETE, DDL, permission changes, or destructive operations
|
||||
against any client/production ERP database unless explicitly authorized in a future policy change.
|
||||
|
||||
Explicit development exception:
|
||||
|
||||
The following database is a disposable TEST DATABASE and is explicitly authorized for writes:
|
||||
|
||||
```
|
||||
D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB
|
||||
```
|
||||
|
||||
For this exact file only, the agent MAY perform:
|
||||
|
||||
- SELECT
|
||||
- INSERT
|
||||
- UPDATE
|
||||
- DELETE
|
||||
- DDL
|
||||
- transactional tests
|
||||
- rollback/recovery tests
|
||||
- destructive development experiments when necessary
|
||||
|
||||
This database has external intact backups and can be restored.
|
||||
|
||||
Safety requirement:
|
||||
|
||||
Before ANY Firebird write, the code must verify that the canonical absolute database path is exactly:
|
||||
|
||||
```
|
||||
D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB
|
||||
```
|
||||
|
||||
If the path differs in any way, writes must fail closed.
|
||||
|
||||
Never infer write authorization from filename alone.
|
||||
Never enable writes to other Firebird databases.
|
||||
|
||||
Secrets:
|
||||
- credentials remain local in ignored environment files;
|
||||
- never log, commit, or expose passwords.
|
||||
|
||||
Development priority:
|
||||
- TESTESOFICIAIS.FDB may be modified freely as needed to validate the MVP.
|
||||
- Do not over-engineer rollback protections for this disposable test database unless they are part
|
||||
of the product behavior being validated.
|
||||
|
||||
## GTI Fiscal write-target rule
|
||||
|
||||
The SGV keeps its fiscal defaults in the canonical product record (`C000025`); every other routine consumes them.
|
||||
GTI Fiscal therefore writes ONLY to the canonical product record, and only fields whose mapping is proven
|
||||
(`docs/SGV_PRODUCT_FISCAL_UI_MAP_V1.json`, `write_target=true`). It never writes to NFCE, NFCE_ITENS, item tables,
|
||||
issued documents or any other transactional/historical table; those are read-only evidence for validation.
|
||||
Company context (UF, CRT) is read from `C000004` at run time — never hardcoded.
|
||||
|
||||
## Local completion mission — exact write allowlist (operator-authorized)
|
||||
|
||||
For the "GTI FISCAL LOCAL COMPLETION V1" mission the operator explicitly authorized writes to the disposable COPIES listed, by exact canonical path, in
|
||||
`docs/LOCAL_WRITE_ALLOWLIST_V1.json` (CONSTRUCAO, MMMODAS, RINCAO, UNIAO under `Projeto\Banco\`, plus TESTESOFICIAIS.FDB). Authorization is by exact
|
||||
path only — never by directory or `.FDB` extension. Any other Firebird under `D:\PROJETOS\GTISISTEMAS` stays strictly READ-ONLY. Writes go through the
|
||||
guarded executor (path allowlist, company-context guard, schema/type/length guard, compare-before-write, rowcount, transaction, audit) and only to
|
||||
`C000025` fields whose contract in `docs/FISCAL_FIELD_CONTRACT_V1.json` has `WRITE_ALLOWED=true`. This authorization does not extend to production or client databases.
|
||||
|
||||
## Production deploy rule (Hub)
|
||||
Normal deploy = `powershell -ExecutionPolicy Bypass -File .\scripts\deploy_hub.ps1 [-PublishAgent -Version X]` (docs/DEPLOY_HUB.md). Do not recreate ad-hoc deploy scripts (deploy115.py etc.) and do not improvise SSH/docker against 10.0.10.117. The script handles VPN/SSH retry, backup, upload, migrations, rebuild, health, release publish, download verification, and automatic rollback; the real error is in `logs/last_deploy_result.json` and the log.
|
||||
|
||||
## Versioning (adopted standard, see docs/VERSIONING.md)
|
||||
|
||||
Hub and Agent are versioned and released independently — never assume they share a version
|
||||
number, and never make the Hub's `GTI_FISCAL_VERSION` bump touch the Agent's own version files
|
||||
(`agent/pyproject.toml`, the `AGENT_VERSION` defaults) or vice versa. The panel's "Atualização
|
||||
disponível" already compares an Agent's reported version against the highest **PUBLISHED**
|
||||
Windows `AgentRelease`, never against the Hub's version — see `docs/VERSIONING.md` for the exact
|
||||
mechanism and the current (informal) compatibility matrix.
|
||||
|
||||
## Permanent mission rules (fiscal completion missions, recorded 2026-09-28)
|
||||
|
||||
- No AI/LLM/embeddings, no paid API, no scraping — every rule/decision is derived from real ERP
|
||||
evidence or an explicit, audited operator/ADMIN decision (`FiscalContextEntry`), never inferred
|
||||
or guessed.
|
||||
- No Firebird triggers or schema changes, ever, for any discovery/completion mission. All queries
|
||||
against any Firebird database (client, lab copy, or disposable test DB) are read-only unless the
|
||||
exact path is on an explicit, mission-scoped write allowlist (see above) — writes still go only
|
||||
through the guarded executor.
|
||||
- Deploy only via `scripts\deploy_hub.ps1` (see "Production deploy rule" above) — never ad-hoc
|
||||
SSH/docker/psql against 10.0.10.117.
|
||||
- If the Agent (agent/) changes in a way that affects its runtime behavior: bump the single-source
|
||||
version, build a new installer, validate it in a fresh Windows Sandbox (install, upgrade in-place,
|
||||
identity preserved), and never overwrite an already-published release. A pure version-label sync
|
||||
bump (release/VERSION, pyproject.toml files, the `AGENT_VERSION` default string) that changes no
|
||||
Agent behavior is not itself an "Agent change" requiring a new installer.
|
||||
- Never reenroll an existing installation (never mint a new `installation_id`/token for a machine
|
||||
that already has one) — an in-place upgrade always preserves identity and local SQLite state.
|
||||
- No remote-execution channel to any client's Agent host, ever (by design — see
|
||||
`docs/rtc_auto_apply.md`). An Agent version upgrade on a real client machine is always performed
|
||||
by a human operator on-site; the assistant never attempts RDP/WinRM/SSH/PowerShell-remoting into
|
||||
a client host, and never asks for or accepts a production `AGENT_TOKEN`/admin session credential
|
||||
pasted in chat to work around that — live validation of a real installation, when needed, comes
|
||||
from the operator (a panel screenshot, or a value they read and paste themselves).
|
||||
- No real client/production data (CNPJ, razão social, product names/descriptions, per-product NCM)
|
||||
is ever published in a location a non-admin can reach (e.g. `central/static/`). Any evidence
|
||||
artifact referenced from a UI card must either be served through an authenticated ADMIN-only
|
||||
route, or be anonymized (generic client labels, no names/documents/descriptions) before being
|
||||
published as a static file.
|
||||
@@ -0,0 +1,41 @@
|
||||
# One of these per Agent installation (one per client machine). Copy to
|
||||
# agent/.env on that machine; never commit it. Get AGENT_ID/AGENT_TOKEN/
|
||||
# COMPANY_ID from central/scripts/register_agent.py, run once per installation.
|
||||
|
||||
GTI_HUB_URL=https://hub.example.com
|
||||
AGENT_ID=
|
||||
AGENT_TOKEN=
|
||||
COMPANY_ID=
|
||||
COMPANY_UF=
|
||||
|
||||
# The client's Firebird ERP database.
|
||||
FIREBIRD_DATABASE_PATH=
|
||||
FB_USER=
|
||||
FB_PASSWORD=
|
||||
FIREBIRD_CLIENT_LIBRARY=
|
||||
|
||||
# Default MUST be 0. Writes only ever happen against exactly the database
|
||||
# named by AUTHORIZED_WRITE_DATABASE_PATH below (never inferred from
|
||||
# FIREBIRD_DATABASE_PATH alone, never from the ".FDB" extension).
|
||||
FIREBIRD_WRITES=0
|
||||
|
||||
# Required to enable writes for THIS installation. Must be set to the exact
|
||||
# same absolute path as FIREBIRD_DATABASE_PATH above — this is the operator's
|
||||
# explicit, per-installation authorization. Leaving it unset means
|
||||
# FIREBIRD_WRITES=1 has no effect against a real client database (it falls
|
||||
# back to the developer-only disposable test database path and therefore
|
||||
# fails closed for anything else).
|
||||
AUTHORIZED_WRITE_DATABASE_PATH=
|
||||
|
||||
# Required to enable writes for THIS installation. Must equal the schema
|
||||
# fingerprint this installation was verified/registered against; if the live
|
||||
# schema fingerprint (reported in heartbeats) ever differs, writes are
|
||||
# refused until an operator re-verifies and updates this value.
|
||||
EXPECTED_SCHEMA_FINGERPRINT=
|
||||
|
||||
POLL_INTERVAL_SECONDS=10
|
||||
AGENT_VERSION=0.1.0
|
||||
|
||||
# Optional overrides:
|
||||
# AGENT_STATE_DB=C:\GTI\Agent\agent_state.sqlite
|
||||
# AGENT_LOG_PATH=C:\GTI\Agent\logs\agent.log
|
||||
@@ -0,0 +1,8 @@
|
||||
[project]
|
||||
name = "gti-fiscal-agent"
|
||||
version = "1.3.9"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = ["httpx>=0.27,<1"]
|
||||
|
||||
[project.optional-dependencies]
|
||||
windows-service = ["pywin32>=306; sys_platform == 'win32'"]
|
||||
@@ -0,0 +1,46 @@
|
||||
"""GTI Fiscal Agent installation preflight (READ-ONLY): python agent/scripts/preflight.py [--env-file FILE] [--json]
|
||||
|
||||
Prints PREFLIGHT=PASS|FAIL and the blockers; exit code 0 on PASS, 2 on FAIL. Secrets are never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
from gti_fiscal_agent.preflight import run_preflight # noqa: E402
|
||||
|
||||
|
||||
def load_env(path):
|
||||
if path and Path(path).exists():
|
||||
for line in Path(path).read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--env-file")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
a = ap.parse_args(argv)
|
||||
load_env(a.env_file or os.environ.get("GTI_AGENT_ENV_FILE") or AGENT_ROOT.parent / ".env")
|
||||
os.environ.setdefault("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
os.environ.setdefault("AGENT_LOG_PATH", str(AGENT_ROOT / "logs" / "agent.log"))
|
||||
res = run_preflight(os.environ)
|
||||
if a.json:
|
||||
print(json.dumps(res, ensure_ascii=False, indent=1, default=str))
|
||||
else:
|
||||
for c in res["checks"]:
|
||||
print(f" [{c['status']:<4}] {c['name']:<24} {c['detail']}")
|
||||
print(f"PREFLIGHT={res['PREFLIGHT']}")
|
||||
for b in res["blockers"]:
|
||||
print(f"BLOCKER: {b}")
|
||||
raise SystemExit(0 if res["PREFLIGHT"] == "PASS" else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Agent preflight: COMPANY_CONTEXT=PASS/FAIL for one Firebird database (READ-ONLY).
|
||||
|
||||
python agent/scripts/preflight_company_context.py [--database <FDB>]
|
||||
|
||||
Configuration (environment or the local ignored .env): COMPANY_CODE / COMPANY_CNPJ (company selection when C000004 is ambiguous),
|
||||
COMPANY_CRT_SOURCE = EMISSOR | C000004 (explicit choice when the two CRT sources disagree). The full CNPJ is never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.company_context import preflight_company_context, preflight_report # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path):
|
||||
if path.exists():
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--database")
|
||||
a = ap.parse_args(argv)
|
||||
load_dotenv(ROOT / ".env")
|
||||
if a.database:
|
||||
os.environ["FIREBIRD_DATABASE_PATH"] = a.database
|
||||
report = preflight_report(preflight_company_context(os.environ))
|
||||
print(json.dumps(report, ensure_ascii=False, indent=1))
|
||||
raise SystemExit(0 if report["COMPANY_CONTEXT"] == "PASS" else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Run the agent against a sanitized sample only after an explicit fingerprint gate."""
|
||||
import argparse, json, os, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
from gti_fiscal_agent.client import HubClient
|
||||
from gti_fiscal_agent.runner import run_sample
|
||||
from gti_fiscal_agent.state import AgentState
|
||||
|
||||
FIELDS = "CODIGO,PRODUTO,NCM,CLASSIFICACAO_FISCAL,CFOP,CFOP_ENTRADA,CST,CST_ENTRADA,ALIQUOTA,CEST,CSOSN,COD_ENTRADA_COFINS,COD_SAIDA_COFINS,ALIQ_COFINS,COD_ENTRADA_PIS,COD_SAIDA_PIS,ALIQ_PIS,ALIQ_IPI,CST_IPI,ORIGEM,IAT,INCIDENCIA_PISCOFINS,IPPT,ULTIMA_ALTERACAO"
|
||||
|
||||
def read_firebird_products(codes):
|
||||
import fdb
|
||||
connection = fdb.connect(database=os.environ["FIREBIRD_DATABASE_PATH"], user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cursor = connection.cursor(); cursor.execute(f"SELECT {FIELDS} FROM C000025 WHERE CODIGO IN ({','.join('?' for _ in codes)}) ORDER BY CODIGO", tuple(codes))
|
||||
names = [column[0].strip() for column in cursor.description]
|
||||
return [dict(zip(names, row)) for row in cursor.fetchall()]
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(); parser.add_argument("--sample", required=True); parser.add_argument("--output", required=True); parser.add_argument("--schema-fingerprint", default="")
|
||||
args = parser.parse_args(); sample = json.loads(Path(args.sample).read_text(encoding="utf-8"))
|
||||
codes = [item["CODIGO"] for item in sample.get("sample_without_ncm", [])[:20]]
|
||||
products = read_firebird_products(codes)
|
||||
result = run_sample(products, HubClient(os.getenv("HUB_URL", "http://localhost:8000"), os.getenv("AGENT_DEV_TOKEN", "")), AgentState(Path(args.output).with_suffix(".sqlite")), args.schema_fingerprint)
|
||||
result["products_requested"] = len(products); result["firebird_products_read"] = len(products); result["firebird_access"] = "READ_ONLY_SELECT"
|
||||
output = Path(args.output)
|
||||
if result["results"]:
|
||||
output.write_text(json.dumps(result, indent=2, default=str), encoding="utf-8")
|
||||
elif not output.exists():
|
||||
output.write_text(json.dumps(result, indent=2, default=str), encoding="utf-8")
|
||||
print(json.dumps(result, indent=2))
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Integrated Agent flow: Firebird read -> Hub /v1/classify/batch -> IPI dry-run
|
||||
-> guarded auto-apply write.
|
||||
|
||||
FIREBIRD_WRITES=0 (default): classifies every product and logs the decision
|
||||
(and what write *would* happen) to the local SQLite audit log; never writes.
|
||||
|
||||
FIREBIRD_WRITES=1: also auto-applies every write_eligible C000025.IPI proposal
|
||||
where rate_kind=NUMERIC, reusing the same schema/path/allowlist/compare-before-write
|
||||
guards as the manual executor. Fails closed before touching the connection unless
|
||||
the configured database is exactly the authorized disposable test database.
|
||||
|
||||
Auto-apply is scoped to table=C000025, field=IPI, rate_kind=NUMERIC only. NT/
|
||||
non-numeric markers, blank rates, ambiguous EX candidates, legacy/invalid NCM,
|
||||
CEST and EX_TIPI are never auto-applied here.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT_ROOT = AGENT_ROOT.parent
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import read_products # noqa: E402
|
||||
from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
|
||||
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, to_hub_product # noqa: E402
|
||||
from gti_fiscal_agent.state import AgentState # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", PROJECT_ROOT / "scripts" / "firebird_write_executor.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--hub-url", default=os.getenv("HUB_URL", "http://localhost:8000"))
|
||||
parser.add_argument("--token", default=os.getenv("AGENT_DEV_TOKEN", ""))
|
||||
parser.add_argument("--state-db", default=str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
parser.add_argument("--schema-fingerprint", default=REFERENCE_SCHEMA_V1)
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(PROJECT_ROOT / ".env")
|
||||
if not args.token:
|
||||
args.token = os.environ.get("AGENT_DEV_TOKEN", "")
|
||||
|
||||
firebird_writes = os.environ.get("FIREBIRD_WRITES", "0") == "1"
|
||||
executor = load_executor()
|
||||
database_path = os.environ["FIREBIRD_DATABASE_PATH"]
|
||||
|
||||
if firebird_writes:
|
||||
# Fail closed before any product is read or any connection is opened for writing.
|
||||
executor.assert_authorized_write_path(database_path)
|
||||
|
||||
if args.schema_fingerprint != REFERENCE_SCHEMA_V1:
|
||||
print(json.dumps({"status": "SCHEMA_UNKNOWN", "processed": 0}))
|
||||
return
|
||||
|
||||
products = read_products()
|
||||
client = HubClient(args.hub_url, args.token)
|
||||
decisions = client.classify_batch([to_hub_product(p) for p in products])
|
||||
|
||||
state = AgentState(args.state_db)
|
||||
|
||||
write_connection = None
|
||||
if firebird_writes:
|
||||
import fdb
|
||||
write_connection = fdb.connect(
|
||||
database=database_path,
|
||||
user=os.environ["FB_USER"],
|
||||
password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
|
||||
try:
|
||||
summary = run_pipeline(products, decisions, state, firebird_writes, write_connection, executor, database_path)
|
||||
finally:
|
||||
if write_connection is not None:
|
||||
write_connection.close()
|
||||
|
||||
summary = {"status": "OK", "firebird_writes_enabled": firebird_writes, **summary}
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,582 @@
|
||||
"""Long-running Agent service: heartbeat + poll Hub jobs + execute pipeline.
|
||||
|
||||
startup -> load config -> validate Hub -> validate Firebird -> heartbeat
|
||||
-> poll job -> claim -> execute -> report result -> back to polling.
|
||||
|
||||
No job available: sleep POLL_INTERVAL_SECONDS.
|
||||
Hub unavailable: retry with simple capped backoff; the Agent stays alive.
|
||||
Job fails internally (bad Firebird read, classify error, write refused):
|
||||
the job is reported FAILED with a sanitized error and the Agent keeps polling.
|
||||
SIGINT/SIGTERM: finish the current job (if any) and stop the loop cleanly.
|
||||
|
||||
Written to be runnable as-is today, and easy to later wrap as a Windows
|
||||
Service (a service wrapper would just call main()); no installer here.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT_ROOT = AGENT_ROOT.parent
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
sys.path.insert(0, str(PROJECT_ROOT)) # ships central/app engine modules read-only (central.app.*)
|
||||
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
from gti_fiscal_agent import firebird_reader as _firebird_reader, rtc_auto_apply as _rtc_apply, unified_fiscal as _unified # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import check_connection, read_products # noqa: E402
|
||||
from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
|
||||
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, product_fingerprint, to_hub_product # noqa: E402
|
||||
from central.app.operational_state import classify_product_state, summarize_operational_states # noqa: E402
|
||||
from gti_fiscal_agent.state import AgentState # noqa: E402
|
||||
from gti_fiscal_agent import enrichment as _enrichment # noqa: E402
|
||||
|
||||
_MAX_BACKOFF_SECONDS = 60
|
||||
SYNC_JOB_KINDS = ("INITIAL_SYNC", "SYNC_INCREMENTAL", "REPROCESS_ALL") # read-only ERP jobs that refresh coverage
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_agent")
|
||||
|
||||
|
||||
def setup_logging(log_path):
|
||||
"""Rotating local log file. Never logs FB_PASSWORD or AGENT_TOKEN — every
|
||||
log call in this module only references non-secret identifiers/counts."""
|
||||
logger.setLevel(logging.INFO)
|
||||
if logger.handlers:
|
||||
return logger
|
||||
Path(log_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
handler = logging.handlers.RotatingFileHandler(log_path, maxBytes=5_000_000, backupCount=5, encoding="utf-8")
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
logger.addHandler(handler)
|
||||
logger.addHandler(logging.StreamHandler())
|
||||
return logger
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def load_executor():
|
||||
# The frozen service host has no repo tree; the installer writes
|
||||
# GTI_WRITE_EXECUTOR_PATH pointing at the extracted executor next to
|
||||
# agent.env. Falls back to the dev/repo path for source runs.
|
||||
executor_path = os.environ.get("GTI_WRITE_EXECUTOR_PATH") or (PROJECT_ROOT / "scripts" / "firebird_write_executor.py")
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", executor_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self, env):
|
||||
self.hub_url = env.get("GTI_HUB_URL") or env.get("HUB_URL") or "http://localhost:8000"
|
||||
self.agent_id = env.get("AGENT_ID", "")
|
||||
self.credential_file = env.get("GTI_CREDENTIAL_FILE") or str(Path(os.environ.get("PROGRAMFILES", r"C:\Program Files")) / "GTI" / "FiscalAgent" / "credentials.bin")
|
||||
# The permanent token NEVER lives in agent.env: it is decrypted from
|
||||
# credentials.bin (DPAPI LocalMachine + SYSTEM/Administrators ACL). An
|
||||
# explicit AGENT_TOKEN/AGENT_DEV_TOKEN in env wins (dev override only).
|
||||
self.agent_token = self._resolve_agent_token(env)
|
||||
self.company_id = env.get("COMPANY_ID", "")
|
||||
self.company_uf = env.get("COMPANY_UF", "")
|
||||
_resolve_firebird_credentials(env)
|
||||
self.database_path = env["FIREBIRD_DATABASE_PATH"]
|
||||
# FISCAL_MODE gates writes technically: DRY_RUN can never write, whatever FIREBIRD_WRITES says. Unset = legacy
|
||||
# behaviour (FIREBIRD_WRITES decides). `firebird_writes` below is the EFFECTIVE value used by every write path.
|
||||
requested = env.get("FIREBIRD_WRITES", "0") == "1"
|
||||
self.fiscal_mode = (env.get("FISCAL_MODE") or ("AUTO_APPLY" if requested else "DRY_RUN")).strip().upper()
|
||||
self.requested_writes = requested
|
||||
self.firebird_writes = requested and self.fiscal_mode == "AUTO_APPLY"
|
||||
# Job execution is controlled CENTRALLY (Hub heartbeat: processing_enabled) — there is no
|
||||
# local toggle, so no installation ever needs its agent.env edited to start/stop work.
|
||||
# (A legacy AGENT_JOBS_ENABLED line in an old agent.env is deliberately ignored.)
|
||||
self.agent_jobs_enabled = True
|
||||
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
|
||||
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
|
||||
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
|
||||
self.agent_version = env.get("AGENT_VERSION", "1.3.9")
|
||||
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
# Pilot write authorization: both must be explicitly set by the installer for
|
||||
# a real client database. Left unset, both default to the dev-only values,
|
||||
# so a real client path/schema is never silently authorized (see
|
||||
# scripts/firebird_write_executor.py's assert_authorized_write_path).
|
||||
self.authorized_write_database_path = env.get("AUTHORIZED_WRITE_DATABASE_PATH")
|
||||
self.expected_schema_fingerprint = env.get("EXPECTED_SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.log_path = env.get("AGENT_LOG_PATH", str(AGENT_ROOT / "logs" / "agent.log"))
|
||||
|
||||
@staticmethod
|
||||
def _resolve_agent_token(env):
|
||||
"""DPAPI credential from credentials.bin; explicit AGENT_TOKEN env wins
|
||||
(dev override). Fail closed: never log or fabricate a token."""
|
||||
from gti_fiscal_agent.credential_store import install_env_from_credentials
|
||||
|
||||
explicit = env.get("AGENT_TOKEN") or env.get("AGENT_DEV_TOKEN") or ""
|
||||
if explicit:
|
||||
return explicit
|
||||
try:
|
||||
return install_env_from_credentials(env.get("GTI_CREDENTIAL_FILE") or str(
|
||||
Path(os.environ.get("PROGRAMFILES", r"C:\Program Files")) / "GTI" / "FiscalAgent" / "credentials.bin"), env)
|
||||
except Exception as exc: # noqa: BLE001 — fail closed: no token, no heartbeat
|
||||
logger.warning("no usable agent credential: %s", exc)
|
||||
return ""
|
||||
|
||||
|
||||
def _resolve_firebird_credentials(env):
|
||||
"""FB_USER/FB_PASSWORD come from the DPAPI-protected firebird.bin written by Setup
|
||||
(never from a hand-edited agent.env, never sent to the Hub). An explicit FB_PASSWORD in
|
||||
the environment still wins (dev override)."""
|
||||
if env.get("FB_PASSWORD"):
|
||||
return
|
||||
from gti_fiscal_agent.credential_store import read_credential
|
||||
|
||||
path = env.get("GTI_FIREBIRD_CREDENTIAL_FILE") or str(Path(os.environ.get("PROGRAMFILES", "C:/Program Files")) / "GTI" / "FiscalAgent" / "firebird.bin")
|
||||
try:
|
||||
data = json.loads(read_credential(path, env=env))
|
||||
except Exception as exc: # noqa: BLE001 — absent/unreadable: preflight will fail closed with a clear blocker
|
||||
logger.info("no Firebird credential file usable: %s", type(exc).__name__)
|
||||
return
|
||||
env["FB_USER"], env["FB_PASSWORD"] = data.get("user", ""), data.get("password", "")
|
||||
|
||||
|
||||
class GracefulStop:
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
signal.signal(signal.SIGINT, self._handle)
|
||||
signal.signal(signal.SIGTERM, self._handle)
|
||||
|
||||
def _handle(self, signum, frame):
|
||||
self.stop = True
|
||||
|
||||
|
||||
def build_job_coverage(env, client, products, decisions):
|
||||
"""Company-context-aware coverage over the job's products (read-only). Fails closed when the company context is unresolved."""
|
||||
from gti_fiscal_agent.company_context import mask_cnpj, preflight_company_context
|
||||
from gti_fiscal_agent.coverage import build_refs, compute_coverage
|
||||
from gti_fiscal_agent.firebird_reader import read_evidence
|
||||
ctx = preflight_company_context(env)
|
||||
if ctx["status"] != "PASS":
|
||||
raise RuntimeError("COMPANY_CONTEXT=FAIL: " + "; ".join(ctx.get("reasons") or ["unresolved"]))
|
||||
evidence = read_evidence(env, ctx["company_code"])
|
||||
refs = build_refs(client.get_json, products, ctx["company_uf"])
|
||||
report = compute_coverage(products, decisions, {"uf": ctx["company_uf"], "crt": ctx["company_crt"]}, evidence.get("company_default_csosn"), refs, evidence)
|
||||
report["company_context"] = {"COMPANY_CONTEXT_STATUS": "PASS", "classification": ctx["classification"], "company_code": ctx["company_code"], "company_cnpj": ctx["company_cnpj"], "company_uf": ctx["company_uf"],
|
||||
"company_crt": ctx["company_crt"], "source_of_truth": ctx["source_of_truth"], "confidence": ctx["confidence"], "warnings": ctx["warnings"]}
|
||||
return report
|
||||
|
||||
|
||||
def open_write_connection(env, database_path, firebird_writes=True):
|
||||
if not firebird_writes: # technical guarantee: no write-capable connection exists in DRY_RUN / FIREBIRD_WRITES=0
|
||||
raise RuntimeError("write connection refused: writes are disabled (FISCAL_MODE=DRY_RUN or FIREBIRD_WRITES=0)")
|
||||
import fdb
|
||||
return fdb.connect(
|
||||
database=database_path,
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
|
||||
|
||||
def _unified_fiscal(env, company, ncm_valid=None):
|
||||
"""PIS/COFINS applicability + RTC (only when the ERP has the reform on), read-only (SELECT only). Fail-closed: if it cannot be assessed the job fails
|
||||
instead of reporting a coverage that ignores these fields."""
|
||||
connection = _firebird_reader._connect(env)
|
||||
try:
|
||||
return _unified.assess_from_connection(connection, company, ncm_valid=ncm_valid)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _fiscal_blocked_products(products, findings):
|
||||
"""One queue entry per (product, blocker). Same shape as every other blocker; `field` is the dashboard category (PIS/COFINS/RTC)."""
|
||||
out = []
|
||||
for product in products:
|
||||
code = str(product.get("CODIGO") or "")
|
||||
found = findings.get(code) or {}
|
||||
for blocker in found.get("blockers") or []:
|
||||
out.append({"company": "", "product_code": code, "gtin": product.get("GTIN") or product.get("EAN_GTIN"), "description": product.get("PRODUTO"), "blocker_type": "BLOCKED_FISCAL_DATA",
|
||||
"field": blocker["category"], "reason_code": blocker["type"], "evidence_summary": {"reason": blocker["reason"], "applicability": found.get("applicability"), **({"human": blocker["human"]} if blocker.get("human") else {})},
|
||||
"input_fingerprint": found.get("input_fingerprint", "")})
|
||||
return out
|
||||
|
||||
|
||||
def _fetch_authorizations(config, job):
|
||||
"""Current central authorization for this installation, straight from the Hub (authenticated). Tests replace this; production never trusts the job's copy for the write itself."""
|
||||
return HubClient(config.hub_url, config.agent_token).authorizations()
|
||||
|
||||
|
||||
def _make_authorizer(config, job, mode, approved_at_claim):
|
||||
"""Re-checks the Hub right before writing (cached for a few seconds): if the mode changed, the product lost its approval, or the Hub cannot be reached, nothing is written (fail closed)."""
|
||||
state = {"at": None, "auth": None}
|
||||
|
||||
def authorize(code):
|
||||
now = time.monotonic()
|
||||
if state["auth"] is None or now - state["at"] > 3.0:
|
||||
try:
|
||||
state["auth"], state["at"] = _fetch_authorizations(config, job) or {}, now
|
||||
except Exception: # noqa: BLE001 - unverifiable authorization is not authorization
|
||||
return "AUTHORIZATION_UNVERIFIABLE"
|
||||
fresh = state["auth"]
|
||||
if fresh.get("rtc_write_mode") != mode:
|
||||
return "AUTHORIZATION_CHANGED"
|
||||
if mode == "APPROVAL_REQUIRED" and str(code) not in {str(c) for c in fresh.get("rtc_approved_codes") or []}:
|
||||
return "AUTHORIZATION_CHANGED"
|
||||
return None
|
||||
return authorize
|
||||
|
||||
|
||||
def _rtc_auto_apply(env, config, job, products, fiscal, company):
|
||||
"""Preview (always) and, only when the Hub-delivered mode permits it, fill CCLASSTRIB for products whose class is PROVEN by an AUTO_APPLY_SAFE rule; also executes
|
||||
Hub-requested rollbacks. Authorization arrives ONLY inside the authenticated job (`authorizations`); there is no local switch. Never raises for a single product."""
|
||||
from datetime import datetime, timezone
|
||||
auth = job.get("authorizations") or {}
|
||||
mode = auth.get("rtc_write_mode") if auth.get("rtc_write_mode") in _rtc_apply.MODES else "DISABLED"
|
||||
approved = {str(c) for c in (auth.get("rtc_approved_codes") or [])}
|
||||
limit = int(getattr(config, "rtc_auto_apply_max_per_job", _rtc_apply.DEFAULT_MAX_PER_JOB))
|
||||
rtc_on = bool(fiscal["context"].get("rtc_enabled"))
|
||||
all_candidates, ready_total = _rtc_apply.select_candidates(fiscal["findings"], products, limit=_rtc_apply.MAX_PREVIEW_CANDIDATES)
|
||||
gate = _rtc_apply.write_gate(mode=mode, capability=True, approved=bool(approved))
|
||||
summary = {"gate": gate, "mode": mode if mode != "DISABLED" else "REPORT_ONLY", "hub_mode": mode, "capability": _rtc_apply.CAPABILITY, "ready_total": ready_total, "max_per_job": limit, "write_attempts": 0, "write_successes": 0,
|
||||
"applied": [], "aborted": [], "aborted_reasons": {}, "rollbacks": [], "allowed_fields": list(_rtc_apply.ALLOWED_RTC_WRITE_FIELDS),
|
||||
"preview": [{k: c[k] for k in ("product_code", "description", "ncm", "current", "proposed", "cst", "rule_id", "basis")} for c in all_candidates] if mode != "DISABLED" and rtc_on else [],
|
||||
"preview_truncated": ready_total > len(all_candidates)}
|
||||
if not rtc_on:
|
||||
return summary
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
rollbacks = [r for r in (auth.get("rtc_rollbacks") or []) if isinstance(r, dict)]
|
||||
if mode == "ENABLED":
|
||||
wanted = all_candidates[:limit]
|
||||
elif mode == "APPROVAL_REQUIRED":
|
||||
wanted = [c for c in all_candidates if c["product_code"] in approved][:limit]
|
||||
else:
|
||||
wanted = []
|
||||
if not (rollbacks or (gate["allowed"] and wanted)):
|
||||
return summary
|
||||
connection = open_write_connection(env, config.database_path)
|
||||
try:
|
||||
if rollbacks:
|
||||
summary["rollbacks"] = _rtc_apply.rollback_records(connection, rollbacks, now=now, job_id=job.get("job_id"))["rollbacks"]
|
||||
if gate["allowed"] and wanted:
|
||||
outcome = _rtc_apply.apply_candidates(connection, wanted, company=company or {}, decision={"local_catalog_hash": fiscal["context"].get("local_catalog_hash"), "origin": f"HUB_MODE:{mode}"}, now=now, job_id=job.get("job_id"), authorize=_make_authorizer(config, job, mode, approved))
|
||||
summary.update({k: outcome[k] for k in ("applied", "aborted", "aborted_reasons", "write_attempts", "write_successes")})
|
||||
finally:
|
||||
connection.close()
|
||||
return summary
|
||||
|
||||
|
||||
def execute_job(job, config, executor, state, env):
|
||||
"""Run the pipeline for one claimed job. Never raises for job-domain
|
||||
failures (bad Firebird read, classify error) — returns (result_summary, error)."""
|
||||
kind = job["kind"]
|
||||
payload = job.get("payload") or {}
|
||||
try:
|
||||
if config.firebird_writes:
|
||||
# Layered, per-installation authorization: the path must match this
|
||||
# installation's own explicitly configured authorized_write_database_path
|
||||
# (defaults to the dev-only constant, so a real client path is refused
|
||||
# unless the installer deliberately set it), AND the live schema
|
||||
# fingerprint must match what this installation was configured to expect.
|
||||
executor.assert_authorized_write_path(config.database_path, authorized_path=config.authorized_write_database_path)
|
||||
executor.assert_schema_fingerprint_authorized(config.schema_fingerprint, config.expected_schema_fingerprint)
|
||||
|
||||
codigos = payload.get("codigos") if kind == "REPROCESS_PRODUCTS" else None
|
||||
products = read_products(env, codigos=codigos)
|
||||
logger.info("job %s: read %d product(s) from Firebird", job.get("job_id"), len(products))
|
||||
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
sync_kind = kind in SYNC_JOB_KINDS
|
||||
cache = sync_kind and hasattr(state, "remember_decision")
|
||||
# SYNC_INCREMENTAL = delta: only products whose source fingerprint changed (or are new) are
|
||||
# re-classified by the Hub; INITIAL_SYNC / REPROCESS_ALL / REPROCESS_PRODUCTS process everything they read.
|
||||
if kind == "SYNC_INCREMENTAL" and cache:
|
||||
to_process = [p for p in products if state.cached_fingerprint(p["CODIGO"]) != product_fingerprint(p)]
|
||||
else:
|
||||
to_process = products
|
||||
decisions = client.classify_batch([to_hub_product(p) for p in to_process])
|
||||
|
||||
write_connection = None
|
||||
if config.firebird_writes:
|
||||
write_connection = open_write_connection(env, config.database_path)
|
||||
try:
|
||||
summary = run_pipeline(to_process, decisions, state, config.firebird_writes, write_connection, executor, config.database_path, authorized_path=config.authorized_write_database_path)
|
||||
finally:
|
||||
if write_connection is not None:
|
||||
write_connection.close()
|
||||
|
||||
operational = coverage = fiscal_context = rtc_apply = None
|
||||
fiscal = None
|
||||
if sync_kind:
|
||||
if cache:
|
||||
for product, decision in zip(to_process, decisions):
|
||||
state.remember_decision(product["CODIGO"], product_fingerprint(product), decision)
|
||||
state.forget_missing([p["CODIGO"] for p in products])
|
||||
all_decisions = [state.cached_decision(p["CODIGO"]) or {} for p in products]
|
||||
else:
|
||||
all_decisions = decisions
|
||||
coverage = build_job_coverage(env, client, products, all_decisions)
|
||||
company = {"crt": ((coverage or {}).get("company_context") or {}).get("company_crt")}
|
||||
ncm_ok = {str(p["CODIGO"]): (d or {}).get("status") == "EXACT_NCM" for p, d in zip(products, all_decisions)}
|
||||
fiscal = _unified_fiscal(env, company, ncm_ok)
|
||||
rtc_apply = _rtc_auto_apply(env, config, job, products, fiscal, company)
|
||||
if rtc_apply["write_successes"] or any(r.get("outcome") == "ROLLED_BACK" for r in rtc_apply["rollbacks"]):
|
||||
fiscal = _unified_fiscal(env, company, ncm_ok) # coverage must reflect the fields that were just filled
|
||||
findings = fiscal["findings"]
|
||||
product_states = [classify_product_state(p, d, fiscal=findings.get(str(p["CODIGO"]))) for p, d in zip(products, all_decisions)]
|
||||
operational = summarize_operational_states(product_states)
|
||||
fiscal_context = {**fiscal["context"], "evaluated_all": kind != "REPROCESS_PRODUCTS"}
|
||||
else:
|
||||
# REPROCESS_PRODUCTS (and any other non-sync kind) used to hardcode company=None here - not a missing feature, a real gap: the SAME
|
||||
# deterministic, fail-closed company_context.resolve() every sync job already uses is cheap (no full-table scan) and applies just as
|
||||
# well to a one-off reprocess job. `company` stays None only when the company genuinely cannot be resolved yet - never a silent guess.
|
||||
try:
|
||||
company = {"crt": _firebird_reader.read_company_context(env).get("crt")}
|
||||
except Exception:
|
||||
company = None
|
||||
fiscal = _unified_fiscal(env, company, {str(p["CODIGO"]): (d or {}).get("status") == "EXACT_NCM" for p, d in zip(to_process, decisions)})
|
||||
fiscal_blocked = _fiscal_blocked_products(products, fiscal["findings"])
|
||||
fiscal_codes = {b["product_code"] for b in fiscal_blocked}
|
||||
summary["blocked_products"] = summary["blocked_products"] + fiscal_blocked
|
||||
summary["resolved_products"] = [c for c in summary["resolved_products"] if str(c) not in fiscal_codes]
|
||||
result_summary = {
|
||||
"coverage": coverage,
|
||||
"operational": operational,
|
||||
"fiscal_context": fiscal_context,
|
||||
"rtc_auto_apply": rtc_apply,
|
||||
"fiscal_mode": getattr(config, "fiscal_mode", None),
|
||||
"release_version": env.get("AGENT_VERSION") or "1.3.9",
|
||||
"products_total": len(products),
|
||||
"products_changed": len(to_process),
|
||||
"products_processed": summary["processed"],
|
||||
"blocked_products": summary["blocked_products"],
|
||||
"resolved_products": summary["resolved_products"],
|
||||
"writes_attempted": summary["writes_attempted"],
|
||||
"writes_succeeded": summary["writes_succeeded"],
|
||||
"writes_failed": summary["writes_failed"],
|
||||
"no_ops": summary["no_ops"],
|
||||
"firebird_writes_enabled": config.firebird_writes,
|
||||
"agent_id": job["agent_id"],
|
||||
"schema_fingerprint": config.schema_fingerprint,
|
||||
}
|
||||
logger.info(
|
||||
"job %s finished: processed=%d writes_attempted=%d writes_succeeded=%d writes_failed=%d no_ops=%d",
|
||||
job.get("job_id"), summary["processed"], summary["writes_attempted"], summary["writes_succeeded"], summary["writes_failed"], summary["no_ops"],
|
||||
)
|
||||
if summary["writes_failed"] > 0:
|
||||
return result_summary, f"{summary['writes_failed']} write(s) failed guard checks"
|
||||
return result_summary, None
|
||||
except executor.WriteRefused as error:
|
||||
logger.warning("job %s refused: %s", job.get("job_id"), error)
|
||||
return None, f"write refused: {error}"
|
||||
except Exception as error: # noqa: BLE001 — job-domain isolation boundary
|
||||
logger.warning("job %s failed internally: %s: %s", job.get("job_id"), type(error).__name__, error)
|
||||
return None, f"{type(error).__name__}: {error}"
|
||||
|
||||
|
||||
BUSY_HEARTBEAT_SECONDS = 20
|
||||
|
||||
|
||||
def _busy_heartbeats(client, installation_id, config, job_id, done):
|
||||
while not done.wait(BUSY_HEARTBEAT_SECONDS):
|
||||
try:
|
||||
client.heartbeat_metadata({"installation_id": installation_id, "status": "BUSY", "schema_fingerprint": config.schema_fingerprint, "agent_version": config.agent_version, "last_job_id": job_id})
|
||||
except Exception: # noqa: BLE001 — Hub hiccup mid-job must not affect the job
|
||||
pass
|
||||
|
||||
|
||||
def resolve_installation_id(config, env, state, client):
|
||||
"""Recover and persist the Hub-bound identity; never invent one for an Agent."""
|
||||
persisted = state.stored_installation_id()
|
||||
configured = (env.get("INSTALLATION_ID") or "").strip() or None
|
||||
if persisted is not None:
|
||||
return state.installation_id(configured)
|
||||
if configured is not None:
|
||||
return state.installation_id(configured)
|
||||
|
||||
configured_agent_id = str(env.get("AGENT_ID") or config.agent_id or "").strip()
|
||||
if not configured_agent_id:
|
||||
raise RuntimeError("AGENT_ID is required to recover installation identity")
|
||||
identity = client.identity()
|
||||
if str(identity.get("agent_id") or "") != configured_agent_id:
|
||||
raise RuntimeError("authenticated identity does not match configured Agent")
|
||||
installation_id = identity.get("installation_id")
|
||||
if not installation_id:
|
||||
raise RuntimeError("authenticated identity response has no installation_id")
|
||||
return state.installation_id(str(installation_id))
|
||||
|
||||
|
||||
def _company_context_for_heartbeat(env, cached):
|
||||
"""Resolves company UF/CRT via the SAME deterministic, fail-closed resolver already used for coverage/RTC (`company_context.resolve`,
|
||||
called through `preflight_company_context`) - never a second, separate guess. Previously the heartbeat only ever sent `COMPANY_UF` from an
|
||||
operator-set env var and never sent CRT at all, so `Installation.company_crt` stayed permanently empty in the Hub even though the Agent
|
||||
already proves CRT from C000004/EMISSOR for every coverage job. Resolves once and caches: never re-queries Firebird once solid, retries only
|
||||
while still unresolved (e.g. the database was not yet reachable when the service started)."""
|
||||
if cached is not None:
|
||||
return cached, None
|
||||
try:
|
||||
from gti_fiscal_agent.company_context import preflight_company_context
|
||||
result = preflight_company_context(env)
|
||||
except Exception as error:
|
||||
return None, f"company context not resolved yet: {error}"
|
||||
if result.get("status") != "PASS":
|
||||
return None, f"company context not resolved yet: {result.get('classification')} ({'; '.join(result.get('reasons', []))})"
|
||||
return result, None
|
||||
|
||||
|
||||
def _start_enrichment(config, env, client, stopper):
|
||||
"""New-product enrichment worker (own thread, own SQLite file). It gates ITSELF on the Hub's per-installation features: DISABLED / not configured = zero Firebird activity."""
|
||||
if not hasattr(client, "resolve_product"):
|
||||
return None
|
||||
store = _enrichment.EnrichmentStore(Path(config.state_db).with_name("enrichment_state.db"))
|
||||
worker = _enrichment.EnrichmentWorker(hub=client, store=store, read_conn=lambda: _firebird_reader._connect(env, True), write_conn=lambda: open_write_connection(env, config.database_path),
|
||||
create_reprocess=lambda codes: client.create_job("REPROCESS_PRODUCTS", {"codigos": list(codes), "reason": "ENRICHMENT"}))
|
||||
threading.Thread(target=worker.run_forever, args=(stopper,), name="product-enrichment", daemon=True).start()
|
||||
return worker
|
||||
|
||||
|
||||
def run(config, env, *, max_iterations=None, stopper=None):
|
||||
"""`stopper` defaults to a signal-based GracefulStop (SIGINT/SIGTERM), which
|
||||
only works from the main thread. A host that manages its own lifecycle
|
||||
(e.g. the Windows Service wrapper, which stops the loop via the Service
|
||||
Control Manager instead of OS signals) should pass its own stop object —
|
||||
anything with a `.stop` boolean attribute — and flip it from any thread."""
|
||||
executor = load_executor()
|
||||
state = AgentState(config.state_db)
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
installation_id = resolve_installation_id(config, env, state, client)
|
||||
stopper = stopper if stopper is not None else GracefulStop()
|
||||
if max_iterations is None:
|
||||
_start_enrichment(config, env, client, stopper)
|
||||
|
||||
backoff = 1
|
||||
was_down = False
|
||||
iterations = 0
|
||||
last_job_id = None
|
||||
company_ctx = None
|
||||
while not stopper.stop:
|
||||
if max_iterations is not None and iterations >= max_iterations:
|
||||
break
|
||||
iterations += 1
|
||||
|
||||
try:
|
||||
company_ctx, reason = _company_context_for_heartbeat(env, company_ctx)
|
||||
if reason:
|
||||
logger.debug(reason)
|
||||
heartbeat_payload = {
|
||||
"installation_id": installation_id, "status": "IDLE", "schema_fingerprint": config.schema_fingerprint,
|
||||
"agent_version": config.agent_version, "last_job_id": last_job_id, "display_name": env.get("AGENT_DISPLAY_NAME", ""),
|
||||
"customer_code": env.get("CUSTOMER_CODE"), "hostname": env.get("AGENT_HOSTNAME") or os.environ.get("COMPUTERNAME"),
|
||||
"windows_version": env.get("WINDOWS_VERSION"), "company_code": config.company_id, "company_uf": (company_ctx or {}).get("company_uf") or config.company_uf,
|
||||
"company_crt": (company_ctx or {}).get("company_crt"), "company_contribuinte_ipi": (company_ctx or {}).get("company_contribuinte_ipi"),
|
||||
"database_path": config.database_path, "capabilities": {"heartbeat": True, "jobs": True, "local_logs": True, "RTC_CCLASSTRIB_WRITE_V1": True, "PRODUCT_ENRICHMENT_WRITE_V1": True},
|
||||
"fiscal_engine_version": env.get("FISCAL_ENGINE_VERSION", config.agent_version), "dataset_versions": {},
|
||||
}
|
||||
if hasattr(client, "heartbeat_metadata"):
|
||||
hb_response = client.heartbeat_metadata(heartbeat_payload)
|
||||
else:
|
||||
hb_response = client.heartbeat("IDLE", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
|
||||
processing_enabled = not (isinstance(hb_response, dict) and hb_response.get("processing_enabled") is False)
|
||||
if was_down:
|
||||
logger.info("reconnected to Hub after %ss backoff", backoff)
|
||||
was_down = False
|
||||
backoff = 1
|
||||
except Exception as error:
|
||||
logger.warning("heartbeat failed, Hub unreachable: %s (retrying in %ss)", error, backoff)
|
||||
was_down = True
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
if not processing_enabled: # central pause: heartbeat keeps flowing, no job is claimed or executed
|
||||
time.sleep(config.poll_interval_seconds)
|
||||
continue
|
||||
|
||||
try:
|
||||
job = client.next_job()
|
||||
except Exception as error:
|
||||
logger.warning("poll failed, Hub unreachable: %s (retrying in %ss)", error, backoff)
|
||||
was_down = True
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
if job is None:
|
||||
time.sleep(config.poll_interval_seconds)
|
||||
continue
|
||||
|
||||
last_job_id = job.get("job_id")
|
||||
logger.info("job %s received: kind=%s", job.get("job_id"), job["kind"])
|
||||
try:
|
||||
if hasattr(client, "heartbeat_metadata"):
|
||||
client.heartbeat_metadata({"installation_id": installation_id, "status": "BUSY", "schema_fingerprint": config.schema_fingerprint, "agent_version": config.agent_version, "last_job_id": last_job_id})
|
||||
else:
|
||||
client.heartbeat("BUSY", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
|
||||
except Exception:
|
||||
pass # best-effort; the job itself still proceeds
|
||||
|
||||
# A full sync can take minutes: keep the heartbeat alive (BUSY) from a side thread so the
|
||||
# panel never shows a working Agent as OFFLINE. The side thread only talks to the Hub
|
||||
# (no SQLite/Firebird), so it cannot interfere with the job.
|
||||
done = threading.Event()
|
||||
beat = threading.Thread(target=_busy_heartbeats, args=(client, installation_id, config, last_job_id, done), daemon=True)
|
||||
beat.start()
|
||||
try:
|
||||
result_summary, error = execute_job(job, config, executor, state, env)
|
||||
finally:
|
||||
done.set()
|
||||
try:
|
||||
if error is None:
|
||||
client.complete_job(job.get("job_id"), result_summary)
|
||||
logger.info("job %s reported COMPLETED", job.get("job_id"))
|
||||
else:
|
||||
client.fail_job(job.get("job_id"), error)
|
||||
logger.warning("job %s reported FAILED: %s", job.get("job_id"), error)
|
||||
except Exception as report_error:
|
||||
logger.warning("could not report job %s result to Hub: %s", job.get("job_id"), report_error)
|
||||
|
||||
|
||||
def enforce_startup_preflight(env):
|
||||
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
|
||||
from gti_fiscal_agent.preflight import run_preflight
|
||||
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.9")
|
||||
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
|
||||
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
|
||||
if report["PREFLIGHT"] != "PASS":
|
||||
logger.error("startup aborted: preflight failed (fail closed): %s", report["blockers"])
|
||||
raise SystemExit(2)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--max-iterations", type=int, default=None, help="stop after N poll cycles (used by tests); omit to run forever")
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(Path(os.environ.get("GTI_AGENT_ENV_FILE") or PROJECT_ROOT / ".env"))
|
||||
config = Config(os.environ)
|
||||
setup_logging(config.log_path)
|
||||
|
||||
logger.info("startup: agent_id=%s company_id=%s agent_version=%s firebird_writes=%s", config.agent_id, config.company_id, config.agent_version, config.firebird_writes)
|
||||
hub_ok = HubClient(config.hub_url, config.agent_token).health()
|
||||
firebird_ok = check_connection(os.environ)
|
||||
logger.info("startup checks: hub_ok=%s firebird_ok=%s", hub_ok, firebird_ok)
|
||||
enforce_startup_preflight(os.environ)
|
||||
|
||||
run(config, os.environ, max_iterations=args.max_iterations)
|
||||
logger.info("shutdown: stopped cleanly")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Windows Service wrapper around run_agent_service.run() — reuses it as-is,
|
||||
does not reimplement any polling/pipeline logic.
|
||||
|
||||
Requires pywin32 (`pip install pywin32`), Windows-only.
|
||||
|
||||
Commands (run from an elevated/administrator terminal):
|
||||
python gti_agent_windows_service.py install # register the service, Startup=Automatic
|
||||
python gti_agent_windows_service.py start
|
||||
python gti_agent_windows_service.py stop
|
||||
python gti_agent_windows_service.py restart
|
||||
python gti_agent_windows_service.py remove # uninstall
|
||||
python gti_agent_windows_service.py --startup=auto --username . --password . install
|
||||
|
||||
Recovery (restart on failure) is not set by pywin32's `install` alone; run once
|
||||
after installing:
|
||||
sc.exe failure "GTIFiscalAgent" reset= 86400 actions= restart/60000/restart/60000/restart/60000
|
||||
|
||||
This module only wraps agent/scripts/run_agent_service.py's `run()` — the
|
||||
actual polling/heartbeat/pipeline logic lives there, unchanged.
|
||||
"""
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(AGENT_ROOT / "scripts"))
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
|
||||
import run_agent_service as agent_service # noqa: E402
|
||||
|
||||
try:
|
||||
import servicemanager
|
||||
import win32event
|
||||
import win32service
|
||||
import win32serviceutil
|
||||
except ImportError: # pragma: no cover — only available on Windows with pywin32 installed
|
||||
servicemanager = win32event = win32service = win32serviceutil = None
|
||||
|
||||
|
||||
class _ServiceStopFlag:
|
||||
"""Plain stop flag (no signal.signal registration) — safe to flip from any
|
||||
thread. The Windows Service Control Manager stops the service, not a
|
||||
console SIGINT/SIGTERM, so this replaces GracefulStop for this host."""
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
|
||||
|
||||
if win32serviceutil is not None:
|
||||
|
||||
class GTIFiscalAgentService(win32serviceutil.ServiceFramework):
|
||||
_svc_name_ = "GTIFiscalAgent"
|
||||
_svc_display_name_ = "GTI Fiscal Agent"
|
||||
_svc_description_ = "Reads the local Firebird ERP, polls the GTI Fiscal Hub for jobs, and applies guarded fiscal updates."
|
||||
|
||||
def __init__(self, args):
|
||||
win32serviceutil.ServiceFramework.__init__(self, args)
|
||||
self.stop_event = win32event.CreateEvent(None, 0, 0, None)
|
||||
self.stop_flag = _ServiceStopFlag()
|
||||
self._thread = None
|
||||
|
||||
def SvcStop(self):
|
||||
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
|
||||
self.stop_flag.stop = True
|
||||
win32event.SetEvent(self.stop_event)
|
||||
|
||||
def SvcDoRun(self):
|
||||
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE, servicemanager.PYS_SERVICE_STARTED, (self._svc_name_, ""))
|
||||
self._thread = threading.Thread(target=self._run_agent, daemon=True)
|
||||
self._thread.start()
|
||||
win32event.WaitForSingleObject(self.stop_event, win32event.INFINITE)
|
||||
|
||||
def _run_agent(self):
|
||||
import os
|
||||
agent_service.load_dotenv(Path(os.environ.get("GTI_AGENT_ENV_FILE") or agent_service.PROJECT_ROOT / ".env"))
|
||||
config = agent_service.Config(os.environ)
|
||||
agent_service.setup_logging(config.log_path)
|
||||
try:
|
||||
agent_service.enforce_startup_preflight(os.environ) # fail closed: a failing preflight stops the service
|
||||
except SystemExit:
|
||||
self.stop_flag.stop = True
|
||||
servicemanager.LogErrorMsg("GTIFiscalAgent: startup preflight failed; see the agent log")
|
||||
win32event.SetEvent(self.stop_event)
|
||||
return
|
||||
try:
|
||||
agent_service.run(config, os.environ, stopper=self.stop_flag)
|
||||
except BaseException as error: # noqa: BLE001 - a dead loop must never look like a healthy "Running" service
|
||||
import logging
|
||||
logging.getLogger("gti_fiscal_agent").exception("fatal error in the Agent loop; stopping the service")
|
||||
servicemanager.LogErrorMsg(f"GTIFiscalAgent: fatal error, stopping: {type(error).__name__}: {error}")
|
||||
self.stop_flag.stop = True
|
||||
win32event.SetEvent(self.stop_event)
|
||||
|
||||
|
||||
def main():
|
||||
if win32serviceutil is None:
|
||||
raise SystemExit("pywin32 is required: pip install pywin32")
|
||||
if len(sys.argv) == 1:
|
||||
# SCM launches a service executable without CLI arguments. HandleCommandLine
|
||||
# requires a command (install/start/debug) and exits with usage when none
|
||||
# is present, which makes SCM report error 1053 for a frozen service host.
|
||||
servicemanager.Initialize()
|
||||
servicemanager.PrepareToHostSingle(GTIFiscalAgentService)
|
||||
servicemanager.StartServiceCtrlDispatcher()
|
||||
else:
|
||||
win32serviceutil.HandleCommandLine(GTIFiscalAgentService)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""Read-only ERP to Fiscal Hub agent."""
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Pure capability detection / schema classification for SGV (C000025/C000004) databases. No I/O, no writes."""
|
||||
|
||||
WRITE_TARGETS = ["CLASSIFICACAO_FISCAL", "CFOP_ENTRADA", "CST_ENTRADA", "CFOP", "CST", "ALIQUOTA", "CEST", "SITUACAO_TRIBUTARIA", "COD_ENTRADA_COFINS",
|
||||
"COD_SAIDA_COFINS", "ALIQ_COFINS", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "CSOSN", "IPI", "CST_IPI", "IAT", "INCIDENCIA_PISCOFINS",
|
||||
"PISCOFINS", "IPPT"]
|
||||
ORIGIN_FIELD = "COD_ORIGEM_MERCADORIA"
|
||||
|
||||
# Firebird RDB$FIELD_TYPE -> family
|
||||
FAMILY = {7: "int", 8: "int", 16: "int", 10: "float", 11: "float", 27: "float", 12: "date", 13: "time", 35: "timestamp", 14: "text", 37: "text", 261: "blob"}
|
||||
# accepted families per field (UI edit controls take text or numbers; numeric columns may be int/float)
|
||||
ACCEPT = {"ALIQUOTA": {"float", "int"}, "IPI": {"float", "int"}, "COD_ORIGEM_MERCADORIA": {"int"}}
|
||||
DEFAULT_ACCEPT = {"text", "int"} # text columns; some ERPs keep codes such as CST_ENTRADA in SMALLINT
|
||||
|
||||
CAPABILITIES = {
|
||||
"supports_ncm": ["CLASSIFICACAO_FISCAL"],
|
||||
"supports_cfop_entry": ["CFOP_ENTRADA"],
|
||||
"supports_cfop_exit": ["CFOP"],
|
||||
"supports_cst_entry": ["CST_ENTRADA"],
|
||||
"supports_cst_exit": ["CST"],
|
||||
"supports_icms_rate": ["ALIQUOTA"],
|
||||
"supports_cest": ["CEST"],
|
||||
"supports_csosn": ["CSOSN"],
|
||||
"supports_ipi": ["IPI", "CST_IPI"],
|
||||
"supports_pis": ["COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS"],
|
||||
"supports_cofins": ["COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS"],
|
||||
"supports_origin": [ORIGIN_FIELD],
|
||||
"supports_iat": ["IAT"],
|
||||
"supports_incidence": ["INCIDENCIA_PISCOFINS"],
|
||||
"supports_ippt": ["IPPT"],
|
||||
}
|
||||
|
||||
|
||||
def type_ok(field, type_code):
|
||||
return FAMILY.get(type_code, "other") in ACCEPT.get(field, DEFAULT_ACCEPT)
|
||||
|
||||
|
||||
def detect_capabilities(columns):
|
||||
"""columns: {FIELD: rdb_type_code} of C000025. capability=true only if every required column exists with a usable type."""
|
||||
out = {}
|
||||
for cap, fields in CAPABILITIES.items():
|
||||
out[cap] = all(f in columns and type_ok(f, columns[f]) for f in fields)
|
||||
return out
|
||||
|
||||
|
||||
def field_compatibility(columns, reference_types=None):
|
||||
"""Per write target (+ origin): PRESENT_OK / PRESENT_TYPE_VARIATION / MISSING."""
|
||||
res = {}
|
||||
for f in WRITE_TARGETS + [ORIGIN_FIELD]:
|
||||
if f not in columns:
|
||||
res[f] = "MISSING"
|
||||
elif not type_ok(f, columns[f]):
|
||||
res[f] = "PRESENT_TYPE_INCOMPATIBLE"
|
||||
elif reference_types and reference_types.get(f) is not None and reference_types[f] != columns[f]:
|
||||
res[f] = "PRESENT_TYPE_VARIATION"
|
||||
else:
|
||||
res[f] = "PRESENT_OK"
|
||||
return res
|
||||
|
||||
|
||||
def classify_schema(has_c000004, has_c000025, fingerprint, reference_fingerprint, compat):
|
||||
"""REFERENCE_SCHEMA_MATCH | COMPATIBLE_WITH_VARIATIONS | INCOMPATIBLE. A different hash is not a failure
|
||||
when every required capability exists (origin is optional: it is only PROBABLE in the map)."""
|
||||
if not (has_c000004 and has_c000025):
|
||||
return "INCOMPATIBLE"
|
||||
required = {f: v for f, v in compat.items() if f in WRITE_TARGETS}
|
||||
if any(v in ("MISSING", "PRESENT_TYPE_INCOMPATIBLE") for v in required.values()):
|
||||
return "INCOMPATIBLE"
|
||||
if fingerprint == reference_fingerprint:
|
||||
return "REFERENCE_SCHEMA_MATCH"
|
||||
return "COMPATIBLE_WITH_VARIATIONS"
|
||||
|
||||
|
||||
NAME_KEYWORDS = {"RESTAURANT": ("RESTAURANTE", "PIZZARIA", "LANCHONETE", "CHURRASC"), "RETAIL_MARKET": ("MERCADO", "SUPERMERC", "MERCEARIA", "ATACAD"),
|
||||
"APPAREL": ("MODAS", "CONFEC", "BOUTIQUE", "ROUPAS"), "CONSTRUCTION": ("CONSTRU", "MATERIAIS"), "AUTO_REPAIR": ("OFICINA", "MECANICA", "AUTO PECAS", "AUTOPECAS")}
|
||||
|
||||
|
||||
def infer_segment(signals, company_name=""):
|
||||
"""Evidence-based segment guess from populated tables/flags (never from the file name).
|
||||
signals: {"restaurant_tables_rows": int, "kitchen_products": int, "recipes": int, "service_orders": int, "vehicle_products": int, "grade_items": int, "nfce_items": int}"""
|
||||
ev = []
|
||||
score = {"RESTAURANT": 0, "AUTO_REPAIR": 0, "APPAREL": 0, "RETAIL_MARKET": 0, "CONSTRUCTION": 0}
|
||||
import unicodedata
|
||||
up = unicodedata.normalize("NFKD", company_name or "").encode("ascii", "ignore").decode().upper()
|
||||
for seg_name, kws in NAME_KEYWORDS.items():
|
||||
hit = [k for k in kws if k in up]
|
||||
if hit:
|
||||
score[seg_name] += 1; ev.append(f"company name in C000004 contains {hit[0]} (weak, name is data, not the file name)")
|
||||
if signals.get("restaurant_tables_rows", 0) > 50: score["RESTAURANT"] += 2; ev.append(f"mesas/comandas rows={signals['restaurant_tables_rows']}")
|
||||
if signals.get("kitchen_products", 0) > 0: score["RESTAURANT"] += 1; ev.append(f"PROD_COZINHA products={signals['kitchen_products']}")
|
||||
if signals.get("recipes", 0) > 0: score["RESTAURANT"] += 1; ev.append(f"recipes={signals['recipes']}")
|
||||
if signals.get("service_orders", 0) > 20: score["AUTO_REPAIR"] += 2; ev.append(f"ordens de serviço={signals['service_orders']}")
|
||||
if signals.get("vehicle_products", 0) > 0: score["AUTO_REPAIR"] += 1; ev.append(f"veículos={signals['vehicle_products']}")
|
||||
if signals.get("grade_items", 0) > 50: score["APPAREL"] += 2; ev.append(f"grade (tamanho/cor) rows={signals['grade_items']}")
|
||||
if signals.get("nfce_items", 0) > 1000 and max(score.values()) == 0: score["RETAIL_MARKET"] += 1; ev.append("high-volume NFC-e retail without restaurant/OS/grade signals")
|
||||
best = max(score, key=score.get)
|
||||
return (best if score[best] > 0 else "UNKNOWN"), score, ev
|
||||
@@ -0,0 +1,185 @@
|
||||
"""CFOP/IPPT auto-apply: fills or corrects ONLY C000025.CFOP and C000025.IPPT, and nothing else,
|
||||
when central.app.field_contract.resolve_product already classified the cell AUTO_FILL_SAFE
|
||||
(state=CHANGE, write_eligible=True) for this product.
|
||||
|
||||
Safety model (defense in depth, default OFF), mirrors gti_fiscal_agent.rtc_auto_apply: the Hub sets
|
||||
a per-installation, per-capability mode DISABLED / DRY_RUN / APPROVAL_REQUIRED / ENABLED and delivers
|
||||
it inside an authenticated job (central.app.cfop_ippt_audit.job_authorizations). The Agent only
|
||||
exposes the restricted capability CFOP_IPPT_WRITE_V1; there is no local .env switch. Write allowlist:
|
||||
ALLOWED_FIELDS = ("CFOP", "IPPT") on C000025 - no free SQL, one short transaction per cell.
|
||||
|
||||
Compare-before-write ("human override sempre vence"): every candidate carries the exact value the
|
||||
field_contract resolver saw when it was selected (`expected_before`). Immediately before the UPDATE
|
||||
the cell is re-read; if it no longer equals `expected_before` - a human (or any other process) changed
|
||||
it since - the write is ABORTED with STALE_VALUE_CHANGED_SINCE_DECISION and nothing is written; if it
|
||||
already equals the proposed value the write is skipped as NO_WRITE_REQUIRED (idempotent - a second run
|
||||
over the same candidates always writes 0). Every applied change is read back and verified; a mismatch
|
||||
never returns APPLIED. `rollback_one` restores the previous value only if the current value still
|
||||
equals what was written (a later human edit blocks the rollback too).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
CAPABILITY = "CFOP_IPPT_WRITE_V1"
|
||||
ALLOWED_FIELDS = ("CFOP", "IPPT")
|
||||
MODES = ("DISABLED", "DRY_RUN", "APPROVAL_REQUIRED", "ENABLED")
|
||||
DEFAULT_MAX_PER_JOB = 500
|
||||
|
||||
|
||||
class CfopIpptApplyAborted(Exception):
|
||||
"""The cell is left untouched; `reason` is reported in the audit."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def write_gate(*, mode: str, capability: bool = True, approved: bool = True) -> dict:
|
||||
"""Same shape/semantics as rtc_auto_apply.write_gate: a real write needs the capability AND an
|
||||
authenticated Hub mode that permits writing (ENABLED, or APPROVAL_REQUIRED for approved cells)."""
|
||||
mode = mode if mode in MODES else "DISABLED"
|
||||
missing = []
|
||||
if not capability:
|
||||
missing.append("CAPABILITY")
|
||||
if mode not in ("ENABLED", "APPROVAL_REQUIRED"):
|
||||
missing.append("HUB_MODE_" + mode)
|
||||
elif mode == "APPROVAL_REQUIRED" and not approved:
|
||||
missing.append("HUB_APPROVAL")
|
||||
return {"allowed": not missing, "missing": missing, "mode": mode, "capability": bool(capability)}
|
||||
|
||||
|
||||
def candidate_fingerprint(code: str, field: str, expected_before, proposed) -> str:
|
||||
import hashlib
|
||||
return hashlib.sha256("|".join((_text(code), field, _text(expected_before), _text(proposed))).encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def select_candidates(results: dict, *, limit: int = DEFAULT_MAX_PER_JOB) -> tuple[list[dict], int]:
|
||||
"""`results` is {product_code: {field: {"state": ..., "current": ..., "proposed": ..., "write_eligible": ..., "reason": ...}}} -
|
||||
the SAME per-field dict central.app.field_contract.resolve_product already returns for every product in the job's dry-run pass.
|
||||
Only CFOP/IPPT cells with state=CHANGE and write_eligible=True are candidates - never a second, independent rule."""
|
||||
from central.app.field_contract import CHANGE
|
||||
ready = []
|
||||
for code, fields in results.items():
|
||||
for field in ALLOWED_FIELDS:
|
||||
r = (fields or {}).get(field) or {}
|
||||
if r.get("state") != CHANGE or not r.get("write_eligible"):
|
||||
continue
|
||||
expected_before, proposed = r.get("current"), r.get("proposed")
|
||||
ready.append({"product_code": code, "field": field, "expected_before": expected_before, "proposed": proposed, "reason": r.get("reason"),
|
||||
"fingerprint": candidate_fingerprint(code, field, expected_before, proposed)})
|
||||
return ready[:limit], len(ready)
|
||||
|
||||
|
||||
def apply_candidate(conn, candidate: dict, *, now: str, job_id=None) -> dict:
|
||||
"""One cell, one short transaction. Raises CfopIpptApplyAborted (nothing written)."""
|
||||
field = candidate["field"]
|
||||
assert field in ALLOWED_FIELDS
|
||||
code, expected_before, proposed = candidate["product_code"], candidate["expected_before"], candidate["proposed"]
|
||||
base = {"product_code": code, "field": field, "job_id": job_id, "timestamp": now, "action": "APPLY", "reason_code": candidate.get("reason")}
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
raise CfopIpptApplyAborted("PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND")
|
||||
current = _text(rows[0][0]) or None
|
||||
norm_expected = _text(expected_before) or None
|
||||
if current == (_text(proposed) or None):
|
||||
raise CfopIpptApplyAborted("NO_WRITE_REQUIRED")
|
||||
if current != norm_expected:
|
||||
raise CfopIpptApplyAborted("STALE_VALUE_CHANGED_SINCE_DECISION")
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND (({field} IS NULL) OR (TRIM({field}) = ?))", (proposed, code, norm_expected or ""))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
raise CfopIpptApplyAborted("UPDATE_ROWCOUNT_NOT_ONE")
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = _text(cur.fetchone()[0]) or None
|
||||
if after != (_text(proposed) or None):
|
||||
raise RuntimeError(f"post-write verification failed for {code}.{field}: {after!r} != {proposed!r}")
|
||||
return {**base, "outcome": "APPLIED", "old_value": current, "new_value": proposed, "rowcount": 1, "verified_after": True}
|
||||
except CfopIpptApplyAborted:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
raise
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def apply_candidates(conn, candidates: list[dict], *, now: str, job_id=None, authorize=None) -> dict:
|
||||
"""Applies each candidate independently; a failure or an abort never stops the rest and never
|
||||
leaves a partial change. `authorize(product_code, field)` (optional) re-asks the Hub, right before
|
||||
each UPDATE, whether the central authorization still permits it."""
|
||||
applied, aborted, reasons = [], [], {}
|
||||
for candidate in candidates:
|
||||
code, field = candidate["product_code"], candidate["field"]
|
||||
if authorize is not None:
|
||||
denied = authorize(code, field)
|
||||
if denied:
|
||||
aborted.append({"outcome": "ABORTED", "product_code": code, "field": field, "reason": denied, "job_id": job_id, "timestamp": now, "action": "APPLY"})
|
||||
reasons[denied] = reasons.get(denied, 0) + 1
|
||||
continue
|
||||
try:
|
||||
applied.append(apply_candidate(conn, candidate, now=now, job_id=job_id))
|
||||
except CfopIpptApplyAborted as stop:
|
||||
aborted.append({"outcome": "ABORTED", "product_code": code, "field": field, "reason": stop.reason, "job_id": job_id, "timestamp": now, "action": "APPLY"})
|
||||
reasons[stop.reason] = reasons.get(stop.reason, 0) + 1
|
||||
except Exception: # noqa: BLE001 - a technical failure of ONE cell must not abort the batch
|
||||
aborted.append({"outcome": "FAILED", "product_code": code, "field": field, "reason": "TECHNICAL_FAILURE", "job_id": job_id, "timestamp": now})
|
||||
reasons["TECHNICAL_FAILURE"] = reasons.get("TECHNICAL_FAILURE", 0) + 1
|
||||
return {"applied": applied, "aborted": aborted, "aborted_reasons": reasons, "write_attempts": len(candidates), "write_successes": len(applied)}
|
||||
|
||||
|
||||
def rollback_one(conn, item: dict, *, now: str, job_id=None) -> dict:
|
||||
"""Restores C000025.<field> to `old_value` for ONE previously applied audit record, only if the
|
||||
current value still equals what was written (`new_value`) - a later human edit blocks the rollback."""
|
||||
field = item.get("field")
|
||||
if field not in ALLOWED_FIELDS:
|
||||
return {"outcome": "ABORTED", "reason": "UNKNOWN_FIELD", "field": field, "job_id": job_id, "timestamp": now}
|
||||
code, written, restore_to = _text(item.get("product_code")), _text(item.get("new_value")) or None, item.get("old_value")
|
||||
base = {"product_code": code, "field": field, "job_id": job_id, "timestamp": now, "action": "ROLLBACK", "rollback_of": item.get("audit_id"), "old_value": written, "new_value": restore_to}
|
||||
if not code:
|
||||
return {**base, "outcome": "ABORTED", "reason": "ROLLBACK_REQUEST_INVALID"}
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
|
||||
current = _text(rows[0][0]) or None
|
||||
if current != written:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "ROLLBACK_CONFLICT_VALUE_CHANGED"}
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND (({field} IS NULL) OR (TRIM({field}) = ?))", (restore_to, code, written or ""))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"}
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = _text(cur.fetchone()[0]) or None
|
||||
if after != (_text(restore_to) or None):
|
||||
raise RuntimeError("post-rollback verification failed")
|
||||
return {**base, "outcome": "ROLLED_BACK", "rowcount": 1, "verified_after": True}
|
||||
except Exception as error: # noqa: BLE001
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return {**base, "outcome": "FAILED", "reason": type(error).__name__}
|
||||
|
||||
|
||||
def rollback_records(conn, items: list[dict], *, now: str, job_id=None) -> dict:
|
||||
results = [rollback_one(conn, item, now=now, job_id=job_id) for item in items]
|
||||
return {"rollbacks": results, "rolled_back": sum(1 for r in results if r["outcome"] == "ROLLED_BACK")}
|
||||
@@ -0,0 +1,85 @@
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
TRANSIENT_STATUS = (502, 503, 504) # reverse proxy / Hub restarting
|
||||
BACKOFF_SECONDS = (2, 5, 10, 20, 30, 30, 30, 30, 30) # 10 attempts, ~3 minutes of outage tolerated mid-job
|
||||
|
||||
|
||||
def _resilient(send):
|
||||
"""Runs one idempotent HTTP call, retrying transport failures (connect/TLS/read/protocol errors) and proxy 502/503/504 with backoff, so a network blip in
|
||||
the middle of a long job does not throw the whole job (and minutes of classification) away. Non-transient errors (4xx) are returned untouched."""
|
||||
last = None
|
||||
for attempt in range(len(BACKOFF_SECONDS) + 1):
|
||||
try:
|
||||
response = send()
|
||||
if response.status_code not in TRANSIENT_STATUS or attempt == len(BACKOFF_SECONDS):
|
||||
return response
|
||||
last = None
|
||||
except httpx.TransportError as error:
|
||||
last = error
|
||||
if attempt == len(BACKOFF_SECONDS):
|
||||
raise
|
||||
time.sleep(BACKOFF_SECONDS[attempt])
|
||||
raise last # pragma: no cover
|
||||
|
||||
|
||||
class HubClient:
|
||||
def __init__(self, url, token, chunk_size=25): self.url, self.token, self.chunk_size = url.rstrip("/"), token, chunk_size
|
||||
def _headers(self): return {"Authorization": f"Bearer {self.token}"}
|
||||
def classify_batch(self, products):
|
||||
results = []
|
||||
for start in range(0, len(products), self.chunk_size):
|
||||
chunk = products[start:start + self.chunk_size]
|
||||
# Small chunks + generous timeout + retry with backoff on any transport error or proxy 5xx: a large first sync must survive
|
||||
# a slow Hub, a reverse-proxy read timeout or a TLS/network blip instead of failing the whole job.
|
||||
response = _resilient(lambda: httpx.post(f"{self.url}/v1/classify/batch", json={"products": chunk}, headers=self._headers(), timeout=120))
|
||||
response.raise_for_status(); results.extend(response.json()["results"])
|
||||
return results
|
||||
def get_json(self, path):
|
||||
response = httpx.get(f"{self.url}{path}", headers=self._headers(), timeout=30)
|
||||
response.raise_for_status(); return response.json()
|
||||
def identity(self): return self.get_json("/v1/agents/me/identity")
|
||||
def authorizations(self): return self.get_json("/v1/agents/me/authorizations")
|
||||
def resolve_product(self, payload):
|
||||
"""Asks the Hub to resolve a NEW product (the Agent never contacts external providers). Bounded: the Hub answers within its own time budget."""
|
||||
response = httpx.post(f"{self.url}/v1/products/resolve", json=payload, headers=self._headers(), timeout=20)
|
||||
response.raise_for_status(); return response.json()
|
||||
def pending_enrichment(self):
|
||||
"""Codes the Hub flagged for a new evaluation (panel button, Reprocessar/Sincronizar, KB decision)."""
|
||||
response = httpx.get(f"{self.url}/v1/products/enrichment/pending", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json().get("codes") or []
|
||||
def report_enrichment(self, payload):
|
||||
response = _resilient(lambda: httpx.post(f"{self.url}/v1/products/enrichment/applied", json=payload, headers=self._headers(), timeout=30))
|
||||
response.raise_for_status(); return response.json()
|
||||
def health(self):
|
||||
response = httpx.get(f"{self.url}/health", timeout=10)
|
||||
return response.status_code == 200
|
||||
def heartbeat(self, status, schema_fingerprint=None, agent_version=None, last_job_id=None):
|
||||
payload = {"status": status, "schema_fingerprint": schema_fingerprint, "agent_version": agent_version, "last_job_id": last_job_id}
|
||||
response = httpx.post(f"{self.url}/v1/agents/heartbeat", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def register(self, payload):
|
||||
response = httpx.post(f"{self.url}/v1/agents/register", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def heartbeat_metadata(self, payload):
|
||||
response = httpx.post(f"{self.url}/v1/agents/heartbeat", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def event(self, payload):
|
||||
response = httpx.post(f"{self.url}/v1/agents/events", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def next_job(self):
|
||||
response = httpx.get(f"{self.url}/v1/jobs/next", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def create_job(self, kind, payload=None):
|
||||
response = httpx.post(f"{self.url}/v1/jobs", json={"kind": kind, "payload": payload or {}}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def get_job(self, job_id):
|
||||
response = httpx.get(f"{self.url}/v1/jobs/{job_id}", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def complete_job(self, job_id, result_summary):
|
||||
response = _resilient(lambda: httpx.post(f"{self.url}/v1/jobs/{job_id}/complete", json={"result_summary": result_summary}, headers=self._headers(), timeout=30))
|
||||
response.raise_for_status(); return response.json()
|
||||
def fail_job(self, job_id, error):
|
||||
response = _resilient(lambda: httpx.post(f"{self.url}/v1/jobs/{job_id}/fail", json={"error": str(error)}, headers=self._headers(), timeout=30))
|
||||
response.raise_for_status(); return response.json()
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Deterministic, auditable company-context resolution for an Agent installation (READ-ONLY).
|
||||
|
||||
Evidence (proved on 4 SGV databases, see docs/COMPANY_CONTEXT_RESOLUTION_V1.json):
|
||||
* EMISSOR.EMITENTE_CNPJ/UF/CRT is the emitter block read by the emission programs (GtiVendas_NFCe.exe and Envio NF-e.exe both bind
|
||||
QrEmissor.EMITENTE_*; neither reads C000004.CRT/OPTANTE_SIMPLES). 100% of emitted NFC-e access keys carry the CNPJ of the C000004 row
|
||||
whose CNPJ equals EMISSOR.EMITENTE_CNPJ, and cUF=29.
|
||||
* C000004.CRT / OPTANTE_SIMPLES belong to the ERP filial registry (SPED, filial screen) and can disagree with EMISSOR.
|
||||
Never guesses: first row / lowest code / latest row are NOT used. Ambiguity -> FAIL with the reason, unless the operator configures
|
||||
COMPANY_CODE or COMPANY_CNPJ (company) and COMPANY_CRT_SOURCE = EMISSOR | C000004 (explicit choice when the two CRT sources conflict).
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
IBGE_UF = {"11": "RO", "12": "AC", "13": "AM", "14": "RR", "15": "PA", "16": "AP", "17": "TO", "21": "MA", "22": "PI", "23": "CE", "24": "RN", "25": "PB", "26": "PE",
|
||||
"27": "AL", "28": "SE", "29": "BA", "31": "MG", "32": "ES", "33": "RJ", "35": "SP", "41": "PR", "42": "SC", "43": "RS", "50": "MS", "51": "MT", "52": "GO",
|
||||
"53": "DF"}
|
||||
VALID_CRT = {"1", "2", "3"}
|
||||
|
||||
|
||||
class CompanyContextError(Exception):
|
||||
def __init__(self, result):
|
||||
super().__init__(f"COMPANY_CONTEXT=FAIL ({result.get('classification')}): {'; '.join(result.get('reasons', []))}")
|
||||
self.result = result
|
||||
|
||||
|
||||
def digits(value):
|
||||
return "".join(c for c in str(value or "") if c.isdigit())
|
||||
|
||||
|
||||
def mask_cnpj(value):
|
||||
d = digits(value)
|
||||
return f"{d[:2]}**********{d[-2:]}" if len(d) >= 4 else None
|
||||
|
||||
|
||||
def cnpj_fingerprint(value):
|
||||
d = digits(value)
|
||||
return hashlib.sha256(d.encode()).hexdigest()[:8] if d else None
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def config_from_env(env):
|
||||
crt_source = (env.get("COMPANY_CRT_SOURCE") or "").strip().upper() or None
|
||||
if crt_source == "AUTO": # AUTO = resolve automatically, only when there is no conflict
|
||||
crt_source = None
|
||||
invalid = False
|
||||
else:
|
||||
invalid = bool(crt_source and crt_source not in ("EMISSOR", "C000004"))
|
||||
uf = (env.get("COMPANY_UF") or "").strip().upper() or None
|
||||
return {"company_code": (env.get("COMPANY_CODE") or "").strip() or None, "company_cnpj": digits(env.get("COMPANY_CNPJ")) or None,
|
||||
"crt_source": crt_source if crt_source in ("EMISSOR", "C000004") else None, "crt_source_invalid": invalid,
|
||||
"uf_override": uf}
|
||||
|
||||
|
||||
def resolve(evidence, config=None):
|
||||
"""evidence: {"c000004": [{code, cnpj, ie, uf, crt, optante, name}], "emissor": [{cnpj, uf, crt}],
|
||||
"nfce_keys": [{"cuf", "cnpj", "n"}], "nf_by_filial": {code: n}}. Pure function."""
|
||||
cfg = config or {}
|
||||
out = {"status": "FAIL", "classification": "UNRESOLVED", "company_code": None, "company_cnpj": None, "company_cnpj_fingerprint": None, "company_uf": None, "company_crt": None,
|
||||
"company_contribuinte_ipi": None, "source_of_truth": {}, "confidence": "NONE", "evidence": [], "warnings": [], "reasons": [], "configured": False}
|
||||
|
||||
def fail(cls, reason):
|
||||
out["classification"] = cls; out["reasons"].append(reason); return out
|
||||
|
||||
if cfg.get("crt_source_invalid"):
|
||||
return fail("CONFIG_REQUIRED", "COMPANY_CRT_SOURCE must be EMISSOR or C000004")
|
||||
companies = [{**c, "cnpj": digits(c.get("cnpj"))} for c in evidence.get("c000004", [])]
|
||||
emissors = [{**e, "cnpj": digits(e.get("cnpj"))} for e in evidence.get("emissor", [])]
|
||||
docs = evidence.get("nfce_keys", [])
|
||||
if not companies:
|
||||
return fail("UNRESOLVED", "C000004 has no company rows")
|
||||
|
||||
# ---- company selection -------------------------------------------------------------------
|
||||
selected = None; basis = None
|
||||
by_code, by_cnpj = cfg.get("company_code"), cfg.get("company_cnpj")
|
||||
if by_code or by_cnpj:
|
||||
out["configured"] = True
|
||||
cand = [c for c in companies if (not by_code or c["code"] == by_code) and (not by_cnpj or c["cnpj"] == by_cnpj)]
|
||||
if len(cand) != 1:
|
||||
return fail("CONFIG_REQUIRED", "configured company (COMPANY_CODE/COMPANY_CNPJ) does not identify exactly one C000004 row")
|
||||
selected, basis = cand[0], "CONFIGURED"
|
||||
elif len(companies) == 1:
|
||||
selected, basis = companies[0], "SINGLE_C000004_ROW"
|
||||
elif len(emissors) == 1 and emissors[0]["cnpj"]:
|
||||
match = [c for c in companies if c["cnpj"] == emissors[0]["cnpj"]]
|
||||
if len(match) != 1:
|
||||
return fail("CONFIG_REQUIRED", f"{len(companies)} C000004 rows and the EMISSOR CNPJ matches {len(match)} of them: configure COMPANY_CODE or COMPANY_CNPJ")
|
||||
selected, basis = match[0], "EMISSOR_CNPJ_MATCH"
|
||||
else:
|
||||
return fail("CONFIG_REQUIRED", f"{len(companies)} C000004 rows and no unique EMISSOR row to disambiguate: configure COMPANY_CODE or COMPANY_CNPJ")
|
||||
out["evidence"].append(f"company selected by {basis}")
|
||||
out.update(company_code=selected["code"], company_cnpj=mask_cnpj(selected["cnpj"]), company_cnpj_fingerprint=cnpj_fingerprint(selected["cnpj"]))
|
||||
out["source_of_truth"]["company"] = basis
|
||||
contribuinte_ipi = _s(selected.get("contribuinte_ipi"))
|
||||
if contribuinte_ipi and contribuinte_ipi.upper() in ("S", "N"): # same row already proven to be THE company; no extra query, no extra trust needed
|
||||
out["company_contribuinte_ipi"] = contribuinte_ipi.upper()
|
||||
out["source_of_truth"]["contribuinte_ipi"] = "C000004"
|
||||
|
||||
# ---- emitter (EMISSOR) must describe the same company --------------------------------------
|
||||
emissor = None
|
||||
if emissors:
|
||||
same = [e for e in emissors if e["cnpj"] and e["cnpj"] == selected["cnpj"]]
|
||||
if same:
|
||||
emissor = same[0]; out["evidence"].append("EMISSOR.EMITENTE_CNPJ equals the selected company CNPJ")
|
||||
else:
|
||||
return fail("CONFLICT", "EMISSOR emitter CNPJ differs from the selected company (unexplained): refusing to guess")
|
||||
else:
|
||||
out["warnings"].append("NO_EMISSOR_ROW")
|
||||
|
||||
# ---- emitted documents must belong to the same CNPJ ------------------------------------------
|
||||
doc_ufs = set()
|
||||
if docs:
|
||||
total = sum(d["n"] for d in docs); own = sum(d["n"] for d in docs if digits(d["cnpj"]) == selected["cnpj"])
|
||||
out["evidence"].append(f"emitted NFC-e keys: {own}/{total} carry the selected CNPJ")
|
||||
if own == 0:
|
||||
return fail("CONFLICT", f"none of the {total} emitted NFC-e keys carries the selected company CNPJ")
|
||||
if own != total: # e.g. a database copy/migration that kept another emitter's history: evidence, not a reason to guess
|
||||
out["warnings"].append(f"HISTORICAL_NFCE_KEYS_FROM_OTHER_CNPJ {total - own}/{total}")
|
||||
doc_ufs = {IBGE_UF.get(d["cuf"]) for d in docs if digits(d["cnpj"]) == selected["cnpj"]} - {None}
|
||||
|
||||
# ---- UF ------------------------------------------------------------------------------------------
|
||||
uf_sources = {}
|
||||
if emissor and _s(emissor.get("uf")): uf_sources["EMISSOR"] = _s(emissor["uf"]).upper()
|
||||
if _s(selected.get("uf")): uf_sources["C000004"] = _s(selected["uf"]).upper()
|
||||
if len(doc_ufs) == 1: uf_sources["DOCUMENT_KEYS"] = next(iter(doc_ufs))
|
||||
if len(doc_ufs) > 1: return fail("CONFLICT", "emitted NFC-e keys carry several UFs")
|
||||
values = set(uf_sources.values())
|
||||
override = cfg.get("uf_override")
|
||||
if len(values) > 1:
|
||||
return fail("CONFLICT", f"UF_SOURCE_CONFLICT {uf_sources}")
|
||||
if override and values and override not in values:
|
||||
return fail("CONFLICT", f"COMPANY_UF={override} differs from the UF proved by the database {sorted(values)}: an override never replaces the resolver silently")
|
||||
if not values and override: # explicit override only when the database proves nothing
|
||||
uf_sources["CONFIG_OVERRIDE"] = override; values = {override}; out["warnings"].append("UF_FROM_EXPLICIT_OVERRIDE")
|
||||
if not values:
|
||||
return fail("UNRESOLVED", "UF_SOURCE_UNKNOWN")
|
||||
out["company_uf"] = next(iter(values)); out["source_of_truth"]["uf"] = "+".join(uf_sources)
|
||||
out["uf_status"] = "UF_SOURCE_CONFIRMED"
|
||||
|
||||
# ---- CRT ------------------------------------------------------------------------------------------
|
||||
e_crt = _s(emissor.get("crt")) if emissor else None
|
||||
c_crt = _s(selected.get("crt"))
|
||||
e_ok, c_ok = e_crt in VALID_CRT, c_crt in VALID_CRT
|
||||
if e_ok and c_ok and e_crt == c_crt:
|
||||
crt, src, status = e_crt, "EMISSOR+C000004", "CRT_SOURCE_CONFIRMED"
|
||||
elif e_ok and c_ok:
|
||||
choice = cfg.get("crt_source")
|
||||
if not choice:
|
||||
out["crt_candidates"] = {"EMISSOR": e_crt, "C000004": c_crt}
|
||||
return fail("CONFLICT", f"CRT_SOURCE_CONFLICT EMISSOR={e_crt} C000004={c_crt}: configure COMPANY_CRT_SOURCE (emission programs read EMISSOR)")
|
||||
crt, src, status = (e_crt if choice == "EMISSOR" else c_crt), f"{choice} (configured; conflict acknowledged)", "CRT_SOURCE_CONFLICT_RESOLVED_BY_CONFIG"
|
||||
out["warnings"].append(f"CRT_CONFLICT EMISSOR={e_crt} C000004={c_crt}")
|
||||
elif e_ok or c_ok:
|
||||
crt, src, status = (e_crt, "EMISSOR", "CRT_SOURCE_SINGLE") if e_ok else (c_crt, "C000004", "CRT_SOURCE_SINGLE")
|
||||
out["warnings"].append("CRT_HAS_A_SINGLE_SOURCE")
|
||||
else:
|
||||
return fail("UNRESOLVED", "CRT_SOURCE_UNKNOWN (no valid CRT in EMISSOR or C000004)")
|
||||
out.update(company_crt=crt, crt_status=status); out["source_of_truth"]["crt"] = src
|
||||
opt = _s(selected.get("optante"))
|
||||
if opt and ((crt in ("1", "2") and opt.upper() != "S") or (crt == "3" and opt.upper() == "S")):
|
||||
out["warnings"].append(f"OPTANTE_SIMPLES={opt} inconsistent with CRT={crt}")
|
||||
|
||||
agree = len(uf_sources) + (2 if status == "CRT_SOURCE_CONFIRMED" else 1)
|
||||
out["confidence"] = "HIGH" if (status == "CRT_SOURCE_CONFIRMED" and len(uf_sources) >= 2) else "MEDIUM"
|
||||
out["status"] = "PASS"
|
||||
out["classification"] = "CONFIGURED" if out["configured"] or status == "CRT_SOURCE_CONFLICT_RESOLVED_BY_CONFIG" else "AUTO_RESOLVED"
|
||||
out["reasons"] = []
|
||||
out["evidence_source_count"] = agree
|
||||
return out
|
||||
|
||||
|
||||
def collect_evidence(cursor):
|
||||
"""Read-only SELECTs. Missing optional tables/columns simply yield less evidence."""
|
||||
cursor.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000004','EMISSOR','NFCE','C000061')")
|
||||
cols = {}
|
||||
for t, f in cursor.fetchall():
|
||||
cols.setdefault(t, set()).add(f)
|
||||
ev = {"c000004": [], "emissor": [], "nfce_keys": [], "nf_by_filial": {}}
|
||||
c4 = cols.get("C000004", set())
|
||||
want = [("CODIGO", "code"), ("CNPJ", "cnpj"), ("IE", "ie"), ("UF", "uf"), ("CRT", "crt"), ("OPTANTE_SIMPLES", "optante"), ("FILIAL", "name"), ("CONTRIBUINTE_IPI", "contribuinte_ipi")]
|
||||
use = [(c, k) for c, k in want if c in c4]
|
||||
if use:
|
||||
cursor.execute(f"SELECT {','.join(c for c, _ in use)} FROM C000004")
|
||||
for row in cursor.fetchall():
|
||||
ev["c000004"].append({k: _s(v) for (_, k), v in zip(use, row)})
|
||||
em = cols.get("EMISSOR", set())
|
||||
want = [("EMITENTE_CNPJ", "cnpj"), ("EMITENTE_UF", "uf"), ("EMITENTE_CRT", "crt")]
|
||||
use = [(c, k) for c, k in want if c in em]
|
||||
if use:
|
||||
cursor.execute(f"SELECT {','.join(c for c, _ in use)} FROM EMISSOR")
|
||||
for row in cursor.fetchall():
|
||||
ev["emissor"].append({k: _s(v) for (_, k), v in zip(use, row)})
|
||||
if "CHAVE" in cols.get("NFCE", set()):
|
||||
cursor.execute("SELECT SUBSTRING(CHAVE FROM 1 FOR 2), SUBSTRING(CHAVE FROM 7 FOR 14), COUNT(*) FROM NFCE WHERE CHAVE IS NOT NULL AND CHAR_LENGTH(TRIM(CHAVE)) = 44 "
|
||||
"GROUP BY SUBSTRING(CHAVE FROM 1 FOR 2), SUBSTRING(CHAVE FROM 7 FOR 14)")
|
||||
ev["nfce_keys"] = [{"cuf": _s(a), "cnpj": _s(b), "n": n} for a, b, n in cursor.fetchall()]
|
||||
if "CODFILIAL" in cols.get("C000061", set()):
|
||||
cursor.execute("SELECT CODFILIAL, COUNT(*) FROM C000061 GROUP BY CODFILIAL")
|
||||
ev["nf_by_filial"] = {_s(a): n for a, n in cursor.fetchall()}
|
||||
return ev
|
||||
|
||||
|
||||
def resolve_from_cursor(cursor, env=None):
|
||||
return resolve(collect_evidence(cursor), config_from_env(env or os.environ))
|
||||
|
||||
|
||||
def preflight_company_context(env=None):
|
||||
"""Opens the configured Firebird database read-only and resolves the company. Never prints the full CNPJ."""
|
||||
from .firebird_reader import _connect
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
return resolve_from_cursor(conn.cursor(), env)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def preflight_report(result):
|
||||
keys = ("company_code", "company_cnpj", "company_uf", "company_crt", "company_contribuinte_ipi", "source_of_truth", "confidence", "classification", "warnings", "reasons")
|
||||
return {"COMPANY_CONTEXT": "PASS" if result["status"] == "PASS" else "FAIL", **{k: result.get(k) for k in keys}}
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Read-only fiscal coverage report for a set of products (used by the Agent's REPROCESS_ALL result and by the multi-database tooling).
|
||||
|
||||
Pure computation over Hub decisions + product rows + local evidence. It reuses the shipped pure engines in central.app
|
||||
(fiscal_decision, product_defaults, icms_context) — no fiscal rule lives here, only guards for the company context:
|
||||
the BA Anexo 1 ST/MVA dataset is applied to UF=BA only and CSOSN logic only under CRT=1.
|
||||
"""
|
||||
from collections import Counter
|
||||
|
||||
PRODUCT_COLUMNS = ("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CFOP", "CSOSN", "CST", "CEST", "IPI", "EX_TIPI", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA")
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def compute_coverage(products, decisions, company, company_default, refs, evidence):
|
||||
"""products/decisions aligned lists. company: {"uf","crt"} (resolved). evidence: {"recipes","composed","purchases","stock_entry"}."""
|
||||
from central.app.fiscal_decision import decide_product
|
||||
from central.app.icms_context import csosn_v2, product_role, st_status, withheld_status
|
||||
from central.app.product_defaults import cfop_default
|
||||
uf_ba = company.get("uf") == "BA"
|
||||
crt1 = company.get("crt") == "1"
|
||||
recipes, composed = evidence.get("recipes", set()), evidence.get("composed", set())
|
||||
purchases, stock_entry = evidence.get("purchases", {}), evidence.get("stock_entry", {})
|
||||
m, cfop_values, situ = Counter(), Counter(), Counter()
|
||||
for p, d in zip(products, decisions):
|
||||
p = {c: p.get(c) for c in PRODUCT_COLUMNS}
|
||||
code = _s(p["CODIGO"]); p["CODIGO"] = code
|
||||
cls = d.get("classification") or {}
|
||||
rec = decide_product(p, d, {"uf": company.get("uf") or "", "crt": company.get("crt") or ""}, refs)
|
||||
ncm_status = d.get("status"); raw_ncm = _s(p["CLASSIFICACAO_FISCAL"])
|
||||
m["NCM_MISSING" if not raw_ncm else ("NCM_VALID" if ncm_status == "EXACT_NCM" else ("NCM_LEGACY" if ncm_status == "LEGACY_NCM" else "NCM_INVALID"))] += 1
|
||||
ipi = rec["IPI"]; rk = (cls.get("ipi") or {}).get("rate_kind"); ist = d.get("ipi_status")
|
||||
if ist == "IPI_RESOLVED":
|
||||
m[{"NUMERIC": "IPI_NUMERIC", "NON_NUMERIC": "IPI_NON_NUMERIC", "BLANK": "IPI_SOURCE_BLANK"}.get(rk, "IPI_NO_MATCH")] += 1
|
||||
elif ist == "IPI_CANDIDATES":
|
||||
m["IPI_AMBIGUOUS"] += 1
|
||||
elif ist != "LEGACY_NCM" and raw_ncm:
|
||||
m["IPI_NO_MATCH"] += 1
|
||||
if ipi["action_class"] == "ALREADY_CORRECT":
|
||||
m["IPI_ALREADY_CORRECT"] += 1
|
||||
if ipi["write_eligible"]:
|
||||
m["IPI_CHANGE_REQUIRED"] += 1; m["IPI_WRITE_ELIGIBLE"] += 1
|
||||
cs_ = d.get("cest_status")
|
||||
if cs_ == "CEST_CONFIRMED":
|
||||
m["CEST_CONFIRMED"] += 1
|
||||
elif cs_ == "CEST_CANDIDATES":
|
||||
m["CEST_CANDIDATES"] += 1
|
||||
cf = cfop_default(p, ncm_status, d.get("ba_st_mva_status") if uf_ba else None, refs["cfop_valid"])
|
||||
cfop_values[_s(p["CFOP"])] += 1
|
||||
m["CFOP_RESOLVED" if cf["status"] == "CFOP_DEFAULT_RESOLVED" else "CFOP_CONTEXT_REQUIRED"] += 1
|
||||
ba = cls.get("ba_st_mva") or {}
|
||||
if uf_ba:
|
||||
st, _ = st_status(ncm_status, d.get("ba_st_mva_status"), cs_, p["CEST"], ba.get("cest"), None)
|
||||
else:
|
||||
st = "ST_UNKNOWN"
|
||||
m[st] += 1
|
||||
wh, _ = withheld_status(purchases.get(code, []))
|
||||
m[{"ST_ALREADY_WITHHELD_CONFIRMED": "ST_WITHHOLDING_CONFIRMED", "ST_ALREADY_WITHHELD_NOT_CONFIRMED": "ST_WITHHOLDING_NOT_CONFIRMED"}.get(wh, "ST_WITHHOLDING_UNKNOWN")] += 1
|
||||
role, _ = product_role(p["TIPO"], p["PROD_COZINHA"] == 1, code in recipes, code in composed, bool(_s(p["PRODUTO_SERVICO"])), bool(purchases.get(code)) or stock_entry.get(code, False))
|
||||
m["PRODUCT_ROLE_KNOWN" if role != "UNKNOWN" else "PRODUCT_ROLE_UNKNOWN"] += 1
|
||||
m["ROLE_" + role] += 1
|
||||
state = csosn_v2(p["CSOSN"], company_default, ncm_status, st if uf_ba else "ST_UNKNOWN", wh, role, refs["csosn"])["status"] if crt1 else "CSOSN_CONTEXT_REQUIRED"
|
||||
m["CSOSN_RESOLVED" if state == "CSOSN_RESOLVED" else ("CSOSN_CANDIDATES" if state == "CSOSN_CANDIDATE" else "CSOSN_CONTEXT_REQUIRED")] += 1
|
||||
situ[_s(p["SITUACAO_TRIBUTARIA"])] += 1
|
||||
n = len(products)
|
||||
fill = lambda f: sum(1 for p in products if _s(p.get(f)) not in (None, "")) # noqa: E731
|
||||
return {"TOTAL_PRODUCTS": n, "metrics": dict(m),
|
||||
"IPI_RESOLVED": m["IPI_NUMERIC"] + m["IPI_NON_NUMERIC"] + m["IPI_SOURCE_BLANK"],
|
||||
"quality": {"ncm_valid_share": round(m["NCM_VALID"] / n, 4) if n else None, "cest_filled": fill("CEST"), "csosn_filled": fill("CSOSN"), "cfop_filled": fill("CFOP"), "ipi_filled": fill("IPI"),
|
||||
"cfop_distinct": len(cfop_values), "cfop_top": cfop_values.most_common(6), "situacao_tributaria": dict(situ.most_common(6)),
|
||||
"purchase_items_C000088": sum(len(v) for v in purchases.values()), "products_with_purchase_evidence": len(purchases), "recipes": len(recipes), "composed_products": len(composed)},
|
||||
"guards_applied": {"st_dataset_ba_only": not uf_ba, "csosn_requires_crt1": not crt1}}
|
||||
|
||||
|
||||
def build_refs(get, products, company_uf):
|
||||
"""Reference-data snapshot from the Hub (`get(path) -> json`). Used by the engines for domain validity only."""
|
||||
refs = {}
|
||||
for key, domain in (("csosn", "CSOSN"), ("cst_icms", "CST_ICMS"), ("cst_pis", "CST_PIS"), ("cst_cofins", "CST_COFINS"), ("origem", "ORIGEM_MERCADORIA")):
|
||||
refs[key] = set(get(f"/v1/reference/fiscal_codes/{domain}")["codes"])
|
||||
valid = set()
|
||||
for code in {_s(p.get(f)) for p in products for f in ("CFOP", "CFOP_ENTRADA") if _s(p.get(f))}:
|
||||
if get(f"/v1/reference/cfop/{code}")["found"]:
|
||||
valid.add(code)
|
||||
refs["cfop_valid"] = valid
|
||||
refs["fcp_uf"] = {}
|
||||
fcp = get(f"/v1/reference/fcp/{company_uf}")
|
||||
if fcp.get("found"):
|
||||
refs["fcp_uf"][company_uf] = fcp
|
||||
refs["cbs_rate_2026"] = get("/v1/reference/rtc_cbs_rate/2026").get("rate")
|
||||
refs["cclasstrib_count"] = sum(d["record_count"] for d in get("/v1/datasets/status") if d["kind"] == "RTC_CCLASSTRIB" and d["status"] == "ACTIVE")
|
||||
return refs
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Windows DPAPI credential store for the GTI Fiscal Agent.
|
||||
|
||||
The permanent Agent credential (AGENT_TOKEN) must NEVER live in `agent.env`
|
||||
(which is world-readable config). It is stored in a dedicated
|
||||
`credentials.bin` file, encrypted with DPAPI in LocalMachine scope
|
||||
(`CRYPTPROTECT_LOCAL_MACHINE`), so any local Windows account on the host can
|
||||
decrypt it — and the file ACL is tightened to SYSTEM/Administrators only via
|
||||
`icacls`. The token is decrypted in-process at runtime by the service host and
|
||||
never logged.
|
||||
|
||||
`write_credential`/`read_credential` are shared by Setup.exe (writes at
|
||||
install time) and the Service host (reads at every start).
|
||||
|
||||
Fallback: a plaintext token file is accepted ONLY in development, guarded by
|
||||
`GTI_ALLOW_PLAINTEXT_CREDENTIAL=1`. Production Setup.exe never enables it.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_PROGRAM_DIR = Path("C:/Program Files/GTI/FiscalAgent")
|
||||
_MAGIC = b"GTIDPAPI1"
|
||||
_DEFAULT_FLAGS = 0x1 | 0x4 # CRYPTPROTECT_UI_FORBIDDEN | CRYPTPROTECT_LOCAL_MACHINE
|
||||
|
||||
|
||||
class CredentialStoreError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class _DpapiLazy:
|
||||
"""Lazy wrapper around win32crypt so importing this module (and the whole
|
||||
agent package) works on non-Windows hosts without pywin32. Functions accept
|
||||
the same signatures pywin32 exposes on this machine's build, tolerating both
|
||||
the tuple and the bytes return shapes seen across pywin32 versions."""
|
||||
|
||||
def __init__(self):
|
||||
self._crypt = None
|
||||
|
||||
def _api(self):
|
||||
if self._crypt is None:
|
||||
try:
|
||||
import win32crypt
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise CredentialStoreError(f"win32crypt unavailable: {exc}") from exc
|
||||
self._crypt = win32crypt
|
||||
return self._crypt
|
||||
|
||||
def protect(self, data):
|
||||
crypt = self._api()
|
||||
returned = crypt.CryptProtectData(data, "gti-fiscal-agent-credential", None, None, None, _DEFAULT_FLAGS)
|
||||
if isinstance(returned, tuple):
|
||||
returned, _ = returned
|
||||
return bytes(returned)
|
||||
|
||||
def unprotect(self, blob):
|
||||
crypt = self._api()
|
||||
returned = crypt.CryptUnprotectData(bytes(blob), None, None, None, 0)
|
||||
if isinstance(returned, tuple):
|
||||
_, returned = returned
|
||||
return bytes(returned)
|
||||
|
||||
|
||||
_dpapi = _DpapiLazy()
|
||||
|
||||
|
||||
def _dpapi_available() -> bool:
|
||||
try:
|
||||
_dpapi._api()
|
||||
return True
|
||||
except CredentialStoreError:
|
||||
return False
|
||||
|
||||
|
||||
def default_credential_path() -> Path:
|
||||
return _PROGRAM_DIR / "credentials.bin"
|
||||
|
||||
|
||||
def _apply_acl(path: Path) -> None:
|
||||
"""Tighten ACL so only SYSTEM and Administrators can read/modify the blob.
|
||||
Well-known SIDs are used (`*S-1-5-18` = SYSTEM, `*S-1-5-32-544` = BUILTIN\
|
||||
Administrators) because account-name mapping is locale-dependent and fails
|
||||
on non-English Windows. Setup.exe runs elevated (`--uac-admin`), so icacls
|
||||
succeeds (the owner — the installing administrator — can change the DACL)."""
|
||||
cmd = ["icacls", str(path), "/inheritance:r", "/grant:r", "*S-1-5-18:(F)", "*S-1-5-32-544:(F)"]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
except FileNotFoundError as exc: # non-Windows host without icacls
|
||||
raise CredentialStoreError(f"icacls unavailable: {exc}") from exc
|
||||
if result.returncode != 0:
|
||||
raise CredentialStoreError(f"icacls failed: {result.stdout} {result.stderr}")
|
||||
|
||||
|
||||
def write_credential(path, payload: str) -> None:
|
||||
"""Encrypt `payload` with DPAPI LocalMachine and write `path` with a
|
||||
SYSTEM/Administrators-only ACL. Fails closed: no plaintext file is ever
|
||||
created on this path."""
|
||||
if isinstance(payload, str):
|
||||
payload = payload.encode("utf-8")
|
||||
if not _dpapi_available():
|
||||
raise CredentialStoreError("DPAPI unavailable; refusing to store the credential as plaintext (fail closed)")
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
blob = _MAGIC + _dpapi.protect(payload)
|
||||
path.write_bytes(blob)
|
||||
try:
|
||||
_apply_acl(path)
|
||||
except CredentialStoreError:
|
||||
path.unlink(missing_ok=True) # do not leave a credential file without its ACL
|
||||
raise
|
||||
|
||||
|
||||
def read_credential(path, *, env=None) -> str:
|
||||
"""Decrypt `path`. A plaintext file is honored ONLY when
|
||||
`GTI_ALLOW_PLAINTEXT_CREDENTIAL=1` is set in `env`/environment (development)."""
|
||||
env = env if env is not None else os.environ
|
||||
path = Path(path)
|
||||
if not path.exists():
|
||||
raise CredentialStoreError(f"credential file missing: {path}")
|
||||
raw = path.read_bytes()
|
||||
if raw.startswith(_MAGIC):
|
||||
return _dpapi.unprotect(raw[len(_MAGIC):]).decode("utf-8")
|
||||
if (env.get("GTI_ALLOW_PLAINTEXT_CREDENTIAL") or "").strip() != "1":
|
||||
raise CredentialStoreError("credential file is plaintext and GTI_ALLOW_PLAINTEXT_CREDENTIAL is not set; refusing (fail closed)")
|
||||
return raw.decode("utf-8").strip()
|
||||
|
||||
|
||||
def install_env_from_credentials(path, env) -> str:
|
||||
"""Fill `env["AGENT_TOKEN"]` from `credentials.bin` unless the env already
|
||||
carries an explicit token (which wins). Returns the effective token —
|
||||
WITHOUT writing it into `agent.env` — and never logs it."""
|
||||
existing = env.get("AGENT_TOKEN") or env.get("AGENT_DEV_TOKEN") or ""
|
||||
if existing:
|
||||
return existing
|
||||
token = read_credential(path, env=env)
|
||||
env["AGENT_TOKEN"] = token
|
||||
return token
|
||||
@@ -0,0 +1,530 @@
|
||||
"""New-product enrichment on the Agent: detect NEW rows of C000025, ask the Hub, write ONLY safe empty fields.
|
||||
|
||||
Safety model (defense in depth; nothing here is switchable locally - the Hub owns the per-installation feature flags):
|
||||
- Feature gate: nothing touches Firebird unless the Hub says features_configured AND enrichment_mode != DISABLED. DRY_RUN never opens a write connection.
|
||||
- Scope: only products that appear AFTER the baseline (taken when the function is first active). The historical catalog is never sent anywhere.
|
||||
- No trigger, no table, no procedure in the SGV database: detection is one `SELECT COUNT(*)` per poll (+ a code diff only when the count moved, and every 60 s as a safety net).
|
||||
- Typed capability PRODUCT_ENRICHMENT_WRITE_V1: ALLOWED_FIELDS is an explicit allowlist on C000025; the only statement is a fixed UPDATE ... WHERE CODIGO=? AND (field IS NULL OR TRIM(field)=''),
|
||||
one short transaction per field, followed by a re-read. COD_ORIGEM_MERCADORIA and cClassTrib are NOT in the list (cClassTrib stays with the existing RTC gates).
|
||||
- Right before EVERY field write: re-read the row, re-check the Hub authorization (fail closed), skip when a human already touched the field.
|
||||
- Human overrides: after writing, the Agent watches the value; anything different from what it wrote (including a deleted value) marks the field HUMAN_OVERRIDDEN forever, is reported to the Hub
|
||||
and is never written again by the enrichment (nor re-suggested by the Hub).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_agent.enrichment")
|
||||
|
||||
CAPABILITY = "PRODUCT_ENRICHMENT_WRITE_V1"
|
||||
ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI")
|
||||
# TECHNICAL_PLACEHOLDER: the SGV refuses a product without NCM, so operators type a filler. It means "not classified yet" (not a human decision) and may be replaced.
|
||||
# Explicit per-field allowlist, identical to central/app/enrichment/placeholders.py (parity test). Nothing else is ever overwritten.
|
||||
PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})} # explicit allowlist; anything else raises before touching SQL
|
||||
# READ-ONLY reporting to the Hub for field_contract.resolve_product's post-NCM fiscal-completion decisions (CFOP/CST/CSOSN/PIS-COFINS/...).
|
||||
# This is NOT a write allowlist - PRODUCT_ENRICHMENT_WRITE_V1 still only ever writes ALLOWED_FIELDS above. Sending the CURRENT value of these
|
||||
# fields (and the classification columns below) just lets the Hub's field_contract engine see what's already filled and never guess.
|
||||
FISCAL_COMPLETION_FIELDS = ("SITUACAO_TRIBUTARIA", "CFOP", "CFOP_ENTRADA", "CST", "CST_ENTRADA", "CSOSN", "ALIQUOTA", "CST_IPI",
|
||||
"COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS",
|
||||
"INCIDENCIA_PISCOFINS", "PISCOFINS", "IAT", "IPPT")
|
||||
# Classification metadata field_contract.infer_role reads to auto-discover product_role (RESALE/OWN_PRODUCTION/INPUT/SERVICE) from real ERP
|
||||
# facts instead of asking the operator. Read-only here too.
|
||||
CLASSIFICATION_CONTEXT_FIELDS = ("CODGRUPO", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO")
|
||||
CURRENT_REPORT_FIELDS = ALLOWED_FIELDS + FISCAL_COMPLETION_FIELDS + CLASSIFICATION_CONTEXT_FIELDS
|
||||
READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN", "CODBARRA") + CURRENT_REPORT_FIELDS # the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty
|
||||
POLL_SECONDS = 2.0
|
||||
SAFETY_DIFF_SECONDS = 60.0
|
||||
OVERRIDE_CHECK_SECONDS = 20.0
|
||||
PENDING_POLL_SECONDS = 10.0 # cheap GET: products the Hub flagged for a new evaluation
|
||||
WATCH_WINDOW_SECONDS = 120.0 # a fresh product is re-read for a while: the operator may add the EAN seconds after the first save
|
||||
RETRY_BACKOFF = (30, 60, 120, 300, 600, 1800)
|
||||
|
||||
|
||||
def _now() -> float:
|
||||
return time.time()
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
class EnrichmentStore:
|
||||
"""Own SQLite connection (the worker runs in its own thread)."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.db = sqlite3.connect(str(path), timeout=10, check_same_thread=False)
|
||||
self.db.row_factory = sqlite3.Row
|
||||
self.db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS enrich_meta(k TEXT PRIMARY KEY, v TEXT);
|
||||
CREATE TABLE IF NOT EXISTS enrich_known(code TEXT PRIMARY KEY);
|
||||
CREATE TABLE IF NOT EXISTS enrich_products(
|
||||
code TEXT PRIMARY KEY, state TEXT NOT NULL, detected_at REAL NOT NULL, next_attempt_at REAL, attempts INTEGER NOT NULL DEFAULT 0,
|
||||
snapshot TEXT, written TEXT NOT NULL DEFAULT '{}', overridden TEXT NOT NULL DEFAULT '{}', last_state TEXT);
|
||||
""")
|
||||
self.db.commit()
|
||||
|
||||
def meta(self, key, default=None):
|
||||
row = self.db.execute("SELECT v FROM enrich_meta WHERE k=?", (key,)).fetchone()
|
||||
return row["v"] if row else default
|
||||
|
||||
def set_meta(self, key, value):
|
||||
self.db.execute("INSERT INTO enrich_meta(k,v) VALUES(?,?) ON CONFLICT(k) DO UPDATE SET v=excluded.v", (key, str(value)))
|
||||
self.db.commit()
|
||||
|
||||
def known(self) -> set[str]:
|
||||
return {r["code"] for r in self.db.execute("SELECT code FROM enrich_known")}
|
||||
|
||||
def add_known(self, codes):
|
||||
self.db.executemany("INSERT OR IGNORE INTO enrich_known(code) VALUES(?)", [(c,) for c in codes])
|
||||
self.db.commit()
|
||||
|
||||
def enqueue(self, code, now=None):
|
||||
now = _now() if now is None else now
|
||||
self.db.execute("INSERT OR IGNORE INTO enrich_products(code,state,detected_at,next_attempt_at) VALUES(?,?,?,?)", (code, "PENDING", now, now))
|
||||
self.db.commit()
|
||||
|
||||
def get(self, code):
|
||||
row = self.db.execute("SELECT * FROM enrich_products WHERE code=?", (code,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def due(self, now):
|
||||
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE state IN ('PENDING','RETRY') AND next_attempt_at<=? ORDER BY detected_at LIMIT 20", (now,))]
|
||||
|
||||
def watching(self, now):
|
||||
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE state IN ('DONE','DRY_RUN_DONE') AND detected_at>=?", (now - WATCH_WINDOW_SECONDS,))]
|
||||
|
||||
def tracked_written(self):
|
||||
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE written != '{}'")]
|
||||
|
||||
def update(self, code, **values):
|
||||
for key in ("written", "overridden"):
|
||||
if key in values and not isinstance(values[key], str):
|
||||
values[key] = json.dumps(values[key])
|
||||
sets = ",".join(f"{k}=?" for k in values)
|
||||
self.db.execute(f"UPDATE enrich_products SET {sets} WHERE code=?", (*values.values(), code))
|
||||
self.db.commit()
|
||||
|
||||
def is_overridden(self, code, field) -> bool:
|
||||
row = self.get(code)
|
||||
return bool(row and json.loads(row["overridden"] or "{}").get(field))
|
||||
|
||||
|
||||
def is_human_overridden(store_path, code, field) -> bool:
|
||||
"""Used by every automatic writer of the Agent (the Fiscal Corrector included) before touching a field the enrichment tracks."""
|
||||
try:
|
||||
return EnrichmentStore(store_path).is_overridden(str(code), field)
|
||||
except Exception: # noqa: BLE001 - no state means nothing is protected yet
|
||||
return False
|
||||
|
||||
|
||||
# ---- Firebird access (injected: `read_conn()` returns a READ-ONLY connection, `write_conn()` a write-capable one, only ever opened by ENABLED mode) ----------------------------------------
|
||||
def is_placeholder(field, value) -> bool:
|
||||
text = _text(value)
|
||||
text = "".join(ch for ch in text if ch.isdigit()) if field == "CLASSIFICACAO_FISCAL" else text
|
||||
return text in PLACEHOLDERS.get(field, frozenset())
|
||||
|
||||
|
||||
def writable_now(field, value) -> bool:
|
||||
return not _text(value) or is_placeholder(field, value)
|
||||
|
||||
|
||||
def _digits(value) -> str:
|
||||
return "".join(ch for ch in _text(value) if ch.isdigit())
|
||||
|
||||
|
||||
def gtin_check_ok(value) -> bool:
|
||||
"""Same acceptance as central.app.enrichment.gtin (8/12/13/14 digits + check digit); parity is asserted by a test. Only used to pick WHICH column to send: the Hub decides."""
|
||||
d = _digits(value)
|
||||
if len(d) not in (8, 12, 13, 14) or len(set(d)) == 1:
|
||||
return False
|
||||
total = sum(int(x) * (3 if i % 2 == 0 else 1) for i, x in enumerate(reversed(d[:-1])))
|
||||
return (10 - total % 10) % 10 == int(d[-1])
|
||||
|
||||
|
||||
def pick_gtin(row) -> str:
|
||||
"""First VALID of GTIN / CODBARRA; otherwise whatever was typed (the Hub classifies it INVALID_GTIN/MISSING_GTIN without calling any provider)."""
|
||||
candidates = [_text(row.get("GTIN")), _text(row.get("CODBARRA"))]
|
||||
return next((c for c in candidates if gtin_check_ok(c)), next((c for c in candidates if c), ""))
|
||||
|
||||
|
||||
def read_codes(conn) -> set[str]:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT CODIGO FROM C000025")
|
||||
codes = {_text(r[0]) for r in cur.fetchall()}
|
||||
conn.commit()
|
||||
return codes
|
||||
|
||||
|
||||
def read_count(conn) -> int:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT COUNT(*) FROM C000025")
|
||||
n = int(cur.fetchone()[0])
|
||||
conn.commit()
|
||||
return n
|
||||
|
||||
|
||||
def read_rows(conn, codes) -> dict[str, dict]:
|
||||
out = {}
|
||||
codes = list(codes)
|
||||
for start in range(0, len(codes), 100):
|
||||
chunk = codes[start:start + 100]
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {','.join(READ_FIELDS)} FROM C000025 WHERE CODIGO IN ({','.join('?' for _ in chunk)})", tuple(chunk))
|
||||
for row in cur.fetchall():
|
||||
rec = dict(zip(READ_FIELDS, row))
|
||||
out[_text(rec["CODIGO"])] = rec
|
||||
conn.commit()
|
||||
return out
|
||||
|
||||
|
||||
def write_field(conn, code: str, field: str, value, observed=None, guard_observed: bool = False) -> dict:
|
||||
"""PRODUCT_ENRICHMENT_WRITE_V1: one field, one short transaction, only if the field is still empty. Returns {outcome, reason?}."""
|
||||
if field not in ALLOWED_FIELDS:
|
||||
raise ValueError(f"field {field!r} is outside the PRODUCT_ENRICHMENT_WRITE_V1 allowlist")
|
||||
if field == "IPI":
|
||||
value = float(value)
|
||||
else:
|
||||
value = _digits(value)
|
||||
if not value or (field == "CLASSIFICACAO_FISCAL" and len(value) != 8) or (field == "CEST" and len(value) != 7):
|
||||
return {"outcome": "ABORTED", "reason": "VALUE_SHAPE_INVALID"}
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
|
||||
raw = rows[0][0]
|
||||
if guard_observed and _text(raw) != _text(observed): # STALE_CONFLICT: the value changed between the Hub decision and now (e.g. a person typed something) -> zero write
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "STALE_CONFLICT"}
|
||||
if not writable_now(field, raw):
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"}
|
||||
if _text(raw): # technical placeholder: compare-and-swap on the exact value just read
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND TRIM({field}) = ?", (value, code, _text(raw)))
|
||||
else:
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (value, code))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"}
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = cur.fetchone()[0]
|
||||
conn.commit()
|
||||
same = (float(after) == value) if field == "IPI" and after is not None else _text(after) == value
|
||||
if not same:
|
||||
return {"outcome": "FAILED", "reason": "POST_WRITE_VERIFICATION_FAILED"}
|
||||
return {"outcome": "APPLIED", "value": value, "rowcount": 1, "replaced_placeholder": _text(raw) if _text(raw) else None}
|
||||
except Exception as error: # noqa: BLE001
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return {"outcome": "FAILED", "reason": type(error).__name__}
|
||||
|
||||
|
||||
def _same_value(field, current, written) -> bool:
|
||||
if field == "IPI":
|
||||
try:
|
||||
return current is not None and _text(current) != "" and abs(float(current) - float(written)) < 1e-9
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return _text(current) == _text(written) and _text(current) != ""
|
||||
|
||||
|
||||
class EnrichmentWorker:
|
||||
def __init__(self, *, hub, store: EnrichmentStore, read_conn, write_conn, create_reprocess=None, clock=_now, sleep=time.sleep):
|
||||
self.hub, self.store, self.read_conn, self.write_conn = hub, store, read_conn, write_conn
|
||||
self.create_reprocess, self.clock, self.sleep = create_reprocess, clock, sleep
|
||||
self._features = None
|
||||
self._features_at = 0.0
|
||||
self._last_count = None
|
||||
self._last_diff = 0.0
|
||||
self._last_override = 0.0
|
||||
self._auth_cache = (None, 0.0)
|
||||
self._count_ms = 0.0
|
||||
self._last_pending = 0.0
|
||||
self._wconn = None
|
||||
self.metrics = {"detected": 0, "resolved": 0, "fields_written": 0, "detect_latency_ms": []}
|
||||
|
||||
def _open(self, factory, attempts=3):
|
||||
"""Firebird can drop a connection under load ('connection lost'): retry briefly instead of aborting the product."""
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
return factory()
|
||||
except Exception: # noqa: BLE001
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
self.sleep(0.3 * (attempt + 1))
|
||||
|
||||
# ---- feature gate ------------------------------------------------------------------------------------------------------------------------------
|
||||
def features(self, max_age=10.0) -> dict:
|
||||
if self._features is None or self.clock() - self._features_at > max_age:
|
||||
try:
|
||||
auth = self.hub.authorizations()
|
||||
self._features = (auth or {}).get("features") or {}
|
||||
except Exception: # noqa: BLE001 - Hub offline: keep the last known gate, never invent one
|
||||
self._features = self._features or {}
|
||||
self._features_at = self.clock()
|
||||
return self._features
|
||||
|
||||
def mode(self) -> str:
|
||||
f = self.features()
|
||||
if not f.get("features_configured"):
|
||||
return "WAITING_FEATURE_CONFIGURATION"
|
||||
return f.get("enrichment_mode") or "DISABLED"
|
||||
|
||||
def _fresh_enabled(self) -> bool:
|
||||
"""Fail-closed authorization read right before a write (cached ~3 s)."""
|
||||
auth, at = self._auth_cache
|
||||
if auth is None or self.clock() - at > 3.0:
|
||||
try:
|
||||
auth = self.hub.authorizations().get("features") or {}
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
self._auth_cache = (auth, self.clock())
|
||||
return bool(auth.get("features_configured")) and auth.get("enrichment_mode") == "ENABLED"
|
||||
|
||||
# ---- detection ---------------------------------------------------------------------------------------------------------------------------------
|
||||
def detect(self) -> list[str]:
|
||||
conn = self._open(self.read_conn)
|
||||
try:
|
||||
if self.store.meta("baseline_done") != "1":
|
||||
codes = read_codes(conn)
|
||||
self.store.add_known(codes) # historical catalog: known, never sent
|
||||
self.store.set_meta("baseline_done", "1")
|
||||
self.store.set_meta("baseline_at", datetime.now(timezone.utc).isoformat())
|
||||
self.store.set_meta("baseline_count", len(codes))
|
||||
self._last_count, self._last_diff = len(codes), self.clock()
|
||||
return []
|
||||
if self.store.meta("rebaseline") == "1":
|
||||
# the function was inactive (DISABLED / WAITING_FEATURE_CONFIGURATION) for a while: whatever appeared meanwhile is NOT "new" for the enrichment
|
||||
# (a long-disabled installation must never send a burst of products to the providers when it is switched on again)
|
||||
codes = read_codes(conn)
|
||||
self.store.add_known(codes)
|
||||
self.store.set_meta("rebaseline", "0")
|
||||
self._last_count, self._last_diff = len(codes), self.clock()
|
||||
return []
|
||||
started = self.clock()
|
||||
t0 = time.monotonic()
|
||||
count = read_count(conn)
|
||||
self._count_ms = (time.monotonic() - t0) * 1000
|
||||
moved = self._last_count is None or count != self._last_count
|
||||
if not moved and self.clock() - self._last_diff < SAFETY_DIFF_SECONDS:
|
||||
return []
|
||||
codes = read_codes(conn)
|
||||
self._last_count, self._last_diff = len(codes), self.clock()
|
||||
new = sorted(codes - self.store.known())
|
||||
self.store.add_known(new)
|
||||
for code in new:
|
||||
self.store.enqueue(code, self.clock())
|
||||
if new:
|
||||
self.metrics["detected"] += len(new)
|
||||
self.metrics["detect_latency_ms"].append(int((self.clock() - started) * 1000))
|
||||
return new
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# ---- one product ---------------------------------------------------------------------------------------------------------------------------------
|
||||
def process(self, item: dict):
|
||||
code, mode = item["code"], self.mode()
|
||||
conn = self._open(self.read_conn)
|
||||
try:
|
||||
rows = read_rows(conn, [code])
|
||||
row = rows.get(code)
|
||||
if row is None or mode not in ("DRY_RUN", "ENABLED"):
|
||||
ev = None
|
||||
else:
|
||||
from .firebird_reader import read_product_evidence # targeted, indexed by product_code - never a full-table scan (see docstring)
|
||||
ev = read_product_evidence(conn, code)
|
||||
finally:
|
||||
conn.close()
|
||||
if row is None:
|
||||
self.store.update(code, state="GONE")
|
||||
return
|
||||
snapshot = f"{pick_gtin(row)}|{_text(row['PRODUTO'])}"
|
||||
if mode not in ("DRY_RUN", "ENABLED"):
|
||||
self.store.update(code, state="SKIPPED_FEATURE", snapshot=snapshot)
|
||||
return
|
||||
attempts = int(item["attempts"] or 0) + 1
|
||||
current = {f: row[f] for f in CURRENT_REPORT_FIELDS}
|
||||
try:
|
||||
result = self.hub.resolve_product({"product_code": code, "gtin": pick_gtin(row), "description": _text(row["PRODUTO"]), "ev": {k: (sorted(v) if isinstance(v, set) else v) for k, v in ev.items()},
|
||||
"current": {k: (None if v is None else str(v)) for k, v in current.items()}})
|
||||
except Exception as error: # noqa: BLE001 - Hub offline / slow: the product stays pending, the SGV is unaffected
|
||||
delay = RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)]
|
||||
self.store.update(code, state="RETRY", attempts=attempts, next_attempt_at=self.clock() + delay, last_state=f"HUB_ERROR:{type(error).__name__}")
|
||||
return
|
||||
self.metrics["resolved"] += 1
|
||||
state = result.get("state")
|
||||
if state == "FEATURE_DISABLED":
|
||||
self.store.update(code, state="SKIPPED_FEATURE", snapshot=snapshot, last_state=state)
|
||||
return
|
||||
if state in ("PENDING_PROVIDER_QUOTA", "PENDING_PROVIDERS_OFFLINE"):
|
||||
retry_at = result.get("retry_at")
|
||||
due = self.clock() + RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)]
|
||||
if retry_at:
|
||||
try:
|
||||
due = max(due, datetime.fromisoformat(retry_at).timestamp())
|
||||
except ValueError:
|
||||
pass
|
||||
self.store.update(code, state="RETRY", attempts=attempts, next_attempt_at=due, last_state=state, snapshot=snapshot)
|
||||
return
|
||||
fields = result.get("fields") or {}
|
||||
reports = []
|
||||
written = json.loads(item.get("written") or "{}")
|
||||
overridden = json.loads(item.get("overridden") or "{}")
|
||||
for field in ALLOWED_FIELDS:
|
||||
decision = fields.get(field) or {}
|
||||
status, value = decision.get("status"), decision.get("value")
|
||||
placeholder = _text(current.get(field)) if is_placeholder(field, current.get(field)) else None
|
||||
base = {"field": field, "rule": decision.get("rule"), "kb_status": decision.get("kb_status"), "evidence": decision.get("evidence") or [], "old": placeholder, "new": value}
|
||||
if status == "HUMAN_OVERRIDDEN" or overridden.get(field):
|
||||
reports.append({**base, "outcome": "HUMAN_OVERRIDDEN", "reason": "HUMAN_OVERRIDDEN", "new": None})
|
||||
continue
|
||||
if status != "AUTO_FILL_SAFE" or value in (None, ""):
|
||||
continue
|
||||
if mode == "DRY_RUN":
|
||||
reports.append({**base, "outcome": "DRY_RUN", "reason": "DRY_RUN_NO_WRITE"})
|
||||
continue
|
||||
outcome = self._write_safely(code, field, value, observed=current.get(field))
|
||||
reports.append({**base, "outcome": outcome["outcome"], "reason": outcome.get("reason")})
|
||||
if outcome["outcome"] == "APPLIED":
|
||||
written[field] = value
|
||||
self.metrics["fields_written"] += 1
|
||||
self.store.update(code, written=written) # remembered at once: a crash right after a write still leaves it protected/tracked
|
||||
self._close_write()
|
||||
self.store.update(code, state="DRY_RUN_DONE" if mode == "DRY_RUN" else "DONE", attempts=attempts, snapshot=snapshot, written=written, last_state=state, next_attempt_at=None)
|
||||
if reports:
|
||||
try:
|
||||
self.hub.report_enrichment({"product_code": code, "fields": reports, "dataset_versions": result.get("dataset_versions") or {}})
|
||||
except Exception: # noqa: BLE001 - audit delivery is retried on the next cycle via last_state
|
||||
self.store.update(code, last_state="AUDIT_PENDING")
|
||||
if written and self.create_reprocess is not None and mode == "ENABLED":
|
||||
try:
|
||||
self.create_reprocess([code]) # cClassTrib is decided by the existing RTC gates, never here
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("reprocess request for %s failed", code)
|
||||
|
||||
def _write_safely(self, code, field, value, observed=None) -> dict:
|
||||
"""Per field: fresh central authorization (fail closed) + guarded typed UPDATE. write_field re-reads the field INSIDE its own transaction and refuses when anything is there."""
|
||||
if not self._fresh_enabled():
|
||||
return {"outcome": "ABORTED", "reason": "AUTHORIZATION_CHANGED"}
|
||||
try:
|
||||
if self._wconn is None:
|
||||
self._wconn = self._open(self.write_conn)
|
||||
return write_field(self._wconn, code, field, value, observed=observed, guard_observed=True)
|
||||
except Exception as error: # noqa: BLE001 - one field failing never breaks the others or the SGV
|
||||
self._close_write()
|
||||
return {"outcome": "FAILED", "reason": type(error).__name__}
|
||||
|
||||
def _close_write(self):
|
||||
conn, self._wconn = self._wconn, None
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
def reevaluate_flagged(self) -> list[str]:
|
||||
"""'Reavaliar' in the panel, Reprocessar/Sincronizar and KB decisions: re-run the normal resolve for tracked, not-yet-enriched products (KB/cache first, the Hub decides about providers)."""
|
||||
try:
|
||||
codes = self.hub.pending_enrichment()
|
||||
except Exception: # noqa: BLE001 - Hub offline: try again next cycle
|
||||
return []
|
||||
done = []
|
||||
for code in codes[:25]:
|
||||
item = self.store.get(code)
|
||||
if item is None:
|
||||
self.store.enqueue(code, self.clock())
|
||||
item = self.store.get(code)
|
||||
try:
|
||||
self.process(item)
|
||||
except Exception as error: # noqa: BLE001
|
||||
self._close_write()
|
||||
logger.warning("re-evaluation of %s failed: %s", code, type(error).__name__)
|
||||
done.append(code)
|
||||
return done
|
||||
|
||||
# ---- human override watch --------------------------------------------------------------------------------------------------------------------------
|
||||
def check_overrides(self):
|
||||
tracked = self.store.tracked_written()
|
||||
if not tracked:
|
||||
return
|
||||
conn = self._open(self.read_conn)
|
||||
try:
|
||||
rows = read_rows(conn, [t["code"] for t in tracked])
|
||||
finally:
|
||||
conn.close()
|
||||
for item in tracked:
|
||||
row = rows.get(item["code"])
|
||||
written, overridden = json.loads(item["written"]), json.loads(item["overridden"] or "{}")
|
||||
new = []
|
||||
for field, value in written.items():
|
||||
if overridden.get(field) or row is None:
|
||||
continue
|
||||
if not _same_value(field, row[field], value): # different value OR blanked/deleted: a person decided, forever
|
||||
overridden[field] = True
|
||||
new.append({"field": field, "outcome": "HUMAN_OVERRIDDEN", "reason": "VALUE_CHANGED_AFTER_WRITE", "old": value, "new": None})
|
||||
if new:
|
||||
self.store.update(item["code"], overridden=overridden)
|
||||
try:
|
||||
self.hub.report_enrichment({"product_code": item["code"], "fields": new})
|
||||
except Exception: # noqa: BLE001 - local protection is already in place; the Hub copy follows on the next cycle
|
||||
logger.warning("override report for %s failed", item["code"])
|
||||
|
||||
# ---- loop ----------------------------------------------------------------------------------------------------------------------------------------
|
||||
def step(self):
|
||||
"""One iteration; returns the list of processed product codes."""
|
||||
mode = self.mode()
|
||||
if mode not in ("DRY_RUN", "ENABLED"):
|
||||
if self.store.meta("baseline_done") == "1":
|
||||
self.store.set_meta("rebaseline", "1")
|
||||
return [] # WAITING_FEATURE_CONFIGURATION / DISABLED: zero Firebird activity
|
||||
self.detect()
|
||||
done = []
|
||||
for item in self.store.due(self.clock()):
|
||||
try:
|
||||
self.process(item)
|
||||
except Exception as error: # noqa: BLE001 - keep the product retryable, never lose the batch
|
||||
self._close_write()
|
||||
attempts = int(item["attempts"] or 0) + 1
|
||||
self.store.update(item["code"], state="RETRY", attempts=attempts, next_attempt_at=self.clock() + RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)], last_state=f"ERROR:{type(error).__name__}")
|
||||
logger.warning("enrichment of %s failed: %s", item["code"], type(error).__name__)
|
||||
done.append(item["code"])
|
||||
for item in self.store.watching(self.clock()): # operator completes the record (EAN typed a moment later): re-resolve once per change
|
||||
rows = None
|
||||
conn = self._open(self.read_conn)
|
||||
try:
|
||||
rows = read_rows(conn, [item["code"]])
|
||||
finally:
|
||||
conn.close()
|
||||
row = (rows or {}).get(item["code"])
|
||||
if row and f"{pick_gtin(row)}|{_text(row['PRODUTO'])}" != item.get("snapshot"):
|
||||
self.process(item)
|
||||
done.append(item["code"])
|
||||
if self.clock() - self._last_pending >= PENDING_POLL_SECONDS:
|
||||
self._last_pending = self.clock()
|
||||
done += self.reevaluate_flagged()
|
||||
if self.clock() - self._last_override >= OVERRIDE_CHECK_SECONDS:
|
||||
self._last_override = self.clock()
|
||||
self.check_overrides()
|
||||
return done
|
||||
|
||||
def poll_seconds(self) -> float:
|
||||
"""Adaptive: the SGV is never loaded by more than ~2.5% of one query-time. 3 s on small catalogs, longer when COUNT(*) is slow (huge catalogs); never above 30 s."""
|
||||
return min(30.0, max(POLL_SECONDS, 40 * self._count_ms / 1000))
|
||||
|
||||
def run_forever(self, stopper):
|
||||
while not stopper.stop:
|
||||
try:
|
||||
self.step()
|
||||
except Exception: # noqa: BLE001 - the enrichment thread must never die or affect the SGV
|
||||
logger.exception("enrichment step failed")
|
||||
self.sleep(self.poll_seconds())
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Shared read-only Firebird access for the Agent: full table or a CODIGO subset."""
|
||||
import os
|
||||
|
||||
FIELDS = "CODIGO,PRODUTO,GTIN AS EAN_GTIN,NCM,CLASSIFICACAO_FISCAL,CFOP,CST,ALIQUOTA,CEST,CSOSN,IPI,EX_TIPI"
|
||||
# Present in every audited SGV database but selected only when the column exists (capability-based, never assumed).
|
||||
OPTIONAL_FIELDS = ("TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA")
|
||||
|
||||
|
||||
def _select_fields(cursor):
|
||||
cursor.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = 'C000025'")
|
||||
have = {r[0] for r in cursor.fetchall()}
|
||||
return FIELDS + "".join(f",{c}" for c in OPTIONAL_FIELDS if c in have)
|
||||
|
||||
|
||||
def _connect(env, read_only=True):
|
||||
import fdb
|
||||
kwargs = dict(
|
||||
database=env["FIREBIRD_DATABASE_PATH"],
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
if read_only:
|
||||
kwargs["isolation_level"] = fdb.ISOLATION_LEVEL_READ_COMMITED_RO
|
||||
return fdb.connect(**kwargs)
|
||||
|
||||
|
||||
def check_connection(env=None):
|
||||
"""Best-effort read-only connectivity probe. Returns True/False, never raises."""
|
||||
env = env or os.environ
|
||||
try:
|
||||
conn = _connect(env)
|
||||
try:
|
||||
conn.cursor().execute("SELECT 1 FROM RDB$DATABASE")
|
||||
finally:
|
||||
conn.close()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def read_products(env=None, codigos=None):
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
fields = _select_fields(cursor)
|
||||
if codigos:
|
||||
placeholders = ",".join("?" for _ in codigos)
|
||||
cursor.execute(f"SELECT {fields} FROM C000025 WHERE CODIGO IN ({placeholders}) ORDER BY CODIGO", tuple(codigos))
|
||||
else:
|
||||
cursor.execute(f"SELECT {fields} FROM C000025 ORDER BY CODIGO")
|
||||
names = [column[0].strip() for column in cursor.description]
|
||||
return [dict(zip(names, row)) for row in cursor.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def parse_company_context(rows):
|
||||
"""Pure: rows of (UF, CRT) from C000004 -> {"uf","crt"}. Fails closed (ValueError) if absent,
|
||||
blank or ambiguous — the engines never fall back to a hardcoded UF/CRT."""
|
||||
pairs = {(str(uf or "").strip().upper(), str(crt or "").strip()) for uf, crt in rows}
|
||||
if len(pairs) != 1:
|
||||
raise ValueError(f"company context unavailable or ambiguous ({len(pairs)} distinct UF/CRT rows in C000004)")
|
||||
uf, crt = next(iter(pairs))
|
||||
if len(uf) != 2 or crt not in ("1", "2", "3"):
|
||||
raise ValueError(f"company context invalid: UF={uf!r} CRT={crt!r}")
|
||||
return {"uf": uf, "crt": crt}
|
||||
|
||||
|
||||
def read_company_context(env=None):
|
||||
"""Resolve the ACTIVE company (UF/CRT) with the deterministic resolver (see company_context.py); fails closed with
|
||||
CompanyContextError when the context is ambiguous, conflicting or unresolved. No hardcoded company values, no first-row fallback."""
|
||||
from .company_context import CompanyContextError, preflight_company_context
|
||||
result = preflight_company_context(env or os.environ)
|
||||
if result["status"] != "PASS":
|
||||
raise CompanyContextError(result)
|
||||
return {"uf": result["company_uf"], "crt": result["company_crt"], "company_code": result["company_code"], "contribuinte_ipi": result.get("company_contribuinte_ipi"),
|
||||
"source_of_truth": result["source_of_truth"], "confidence": result["confidence"]}
|
||||
|
||||
|
||||
def read_product_evidence(conn, code):
|
||||
"""Targeted, single-product evidence for an ad-hoc /v1/products/resolve call (new-product enrichment) - the same facts `read_evidence`
|
||||
already gathers for a full sync job (recipes/composed/purchases/stock), but scoped to ONE product_code via an indexed WHERE, never a
|
||||
full-table scan. Measured on a real production-equivalent database: read_evidence's full scan took ~40ms total for the whole catalog,
|
||||
so this is not a correctness-driven optimization - it is what the mission explicitly asked for (never transport/scan the whole dataset
|
||||
for one ad-hoc product). Every table/column is optional: a missing one just means less evidence (capability-based), matching
|
||||
`read_evidence`. Takes an already-open connection (the SAME one the caller already opened for `read_rows`) - never opens its own,
|
||||
so it works with the EnrichmentWorker's injected `read_conn` (including test fakes; a non-Firebird connection has no RDB$RELATION_FIELDS,
|
||||
so the capability probe below degrades to "no evidence" rather than raising)."""
|
||||
from collections import defaultdict
|
||||
cur = conn.cursor()
|
||||
cols = defaultdict(set)
|
||||
try:
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000032','C000088','C000100','RECEITA_INGREDIENTES')")
|
||||
for t, f in cur.fetchall():
|
||||
cols[t].add(f)
|
||||
except Exception:
|
||||
return {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
|
||||
has = lambda t, *c: t in cols and all(x in cols[t] for x in c) # noqa: E731
|
||||
s = lambda v: None if v is None else str(v).strip() # noqa: E731
|
||||
ev = {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
|
||||
code_padded = str(code).zfill(6)
|
||||
if has("RECEITA_INGREDIENTES", "CODRECEITA"):
|
||||
cur.execute("SELECT FIRST 1 CODRECEITA FROM RECEITA_INGREDIENTES WHERE CODRECEITA = ?", (code_padded,))
|
||||
if cur.fetchone():
|
||||
ev["recipes"] = {code_padded}
|
||||
if has("C000032", "CODPRODUTO", "TIPO"):
|
||||
# Counted, never a bare "ever happened" check: a single COMPOSICAO row among many VEND rows
|
||||
# (e.g. a beer occasionally used as a recipe component) is not proof the product IS own
|
||||
# production - central.app.field_contract.infer_role needs the ratio to judge that.
|
||||
cur.execute("SELECT TRIM(TIPO), COUNT(*) FROM C000032 WHERE CODPRODUTO = ? GROUP BY 1", (code,))
|
||||
counts = {(t or ""): n for t, n in cur.fetchall()}
|
||||
composed_n = counts.get("COMPOSICAO", 0)
|
||||
other_n = sum(n for t, n in counts.items() if t != "COMPOSICAO")
|
||||
if composed_n or other_n:
|
||||
ev["composed_ratio"][s(code)] = {"composed": composed_n, "other": other_n}
|
||||
if composed_n:
|
||||
ev["composed"] = {s(code)}
|
||||
if has("C000088", "CODPRODUTO", "ICMS_RETIDO", "ICMS_VALORRETIDO", "SUB_VALOR"):
|
||||
cur.execute("SELECT ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR FROM C000088 WHERE CODPRODUTO = ?", (code,))
|
||||
rows = [{"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv} for ret, vr, sv in cur.fetchall()]
|
||||
if rows:
|
||||
ev["purchases"][s(code)] = rows
|
||||
if has("C000100", "CODPRODUTO", "ULTIMA_ENTRADA", "NOTA_FISCAL"):
|
||||
cur.execute("SELECT FIRST 1 ULTIMA_ENTRADA, NOTA_FISCAL FROM C000100 WHERE CODPRODUTO = ?", (code,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
ev["stock_entry"][s(code)] = row[0] is not None or s(row[1]) not in (None, "", "000000")
|
||||
if has("C000100", "CODPRODUTO", "ENTRADAS"):
|
||||
cur.execute("SELECT FIRST 1 ENTRADAS FROM C000100 WHERE CODPRODUTO = ?", (code,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
ev["stock_in"][s(code)] = bool(row[0] and float(row[0]) > 0)
|
||||
return ev
|
||||
|
||||
|
||||
def read_evidence(env=None, selected_company_code=None):
|
||||
"""Read-only evidence tables used by the coverage report (purchases, stock entries, recipes, compositions) and the company's
|
||||
default CSOSN. Every table/column is optional: a missing one just means less evidence (capability-based)."""
|
||||
from collections import defaultdict
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000004','C000032','C000088','C000100','RECEITA_INGREDIENTES')")
|
||||
cols = defaultdict(set)
|
||||
for t, f in cur.fetchall():
|
||||
cols[t].add(f)
|
||||
has = lambda t, *c: t in cols and all(x in cols[t] for x in c) # noqa: E731
|
||||
s = lambda v: None if v is None else str(v).strip() # noqa: E731
|
||||
ev = {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": defaultdict(list), "stock_entry": {}, "stock_in": {}, "company_default_csosn": None}
|
||||
if has("RECEITA_INGREDIENTES", "CODRECEITA"):
|
||||
cur.execute("SELECT CODRECEITA FROM RECEITA_INGREDIENTES WHERE CODRECEITA IS NOT NULL")
|
||||
ev["recipes"] = {str(r[0]).zfill(6) for r in cur.fetchall()}
|
||||
if has("C000032", "CODPRODUTO", "TIPO"):
|
||||
# Counted per product, never a bare "ever happened" check - see read_product_evidence for why
|
||||
# (a single COMPOSICAO row among many VEND rows is not proof of own production).
|
||||
cur.execute("SELECT CODPRODUTO, TRIM(TIPO), COUNT(*) FROM C000032 GROUP BY 1, 2")
|
||||
counts = defaultdict(lambda: {"composed": 0, "other": 0})
|
||||
for cod, tipo, n in cur.fetchall():
|
||||
counts[s(cod)]["composed" if tipo == "COMPOSICAO" else "other"] += n
|
||||
ev["composed_ratio"] = dict(counts)
|
||||
ev["composed"] = {code for code, c in counts.items() if c["composed"]}
|
||||
if has("C000088", "CODPRODUTO", "ICMS_RETIDO", "ICMS_VALORRETIDO", "SUB_VALOR"):
|
||||
cur.execute("SELECT CODPRODUTO, ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR FROM C000088")
|
||||
for cod, ret, vr, sv in cur.fetchall():
|
||||
ev["purchases"][s(cod)].append({"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv})
|
||||
if has("C000100", "CODPRODUTO", "ULTIMA_ENTRADA", "NOTA_FISCAL"):
|
||||
cur.execute("SELECT CODPRODUTO, ULTIMA_ENTRADA, NOTA_FISCAL FROM C000100")
|
||||
ev["stock_entry"] = {s(r[0]): (r[1] is not None or s(r[2]) not in (None, "", "000000")) for r in cur.fetchall()}
|
||||
if has("C000100", "CODPRODUTO", "ENTRADAS"):
|
||||
cur.execute("SELECT CODPRODUTO, ENTRADAS FROM C000100")
|
||||
ev["stock_in"] = {s(r[0]): bool(r[1] and float(r[1]) > 0) for r in cur.fetchall()}
|
||||
if has("C000004", "ICMS_CSOSN", "CODIGO"):
|
||||
cur.execute("SELECT CODIGO, ICMS_CSOSN FROM C000004")
|
||||
rows = {s(a): s(b) for a, b in cur.fetchall()}
|
||||
ev["company_default_csosn"] = rows.get(selected_company_code) if selected_company_code else (next(iter(rows.values())) if len(rows) == 1 else None)
|
||||
ev["purchases"] = dict(ev["purchases"])
|
||||
return ev
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Decide whether a classified product's IPI rate may be auto-applied.
|
||||
|
||||
Scope is deliberately narrow: C000025.IPI only, and only when the Hub's TIPI
|
||||
match is a single unambiguous numeric rate. Never auto-applies NT/non-numeric
|
||||
markers, blank source rates, ambiguous EX candidates, or legacy/invalid NCM.
|
||||
"""
|
||||
|
||||
|
||||
def ipi_write_decision(decision, current_ipi):
|
||||
"""current_ipi must be a value freshly read from Firebird immediately before
|
||||
this call — None here means the ERP field is genuinely NULL right now, not
|
||||
that it wasn't read."""
|
||||
if decision.get("status") == "LEGACY_NCM":
|
||||
return {"eligible": False, "reason_code": "LEGACY_NCM", "proposed_value": None}
|
||||
if decision.get("ipi_status") != "IPI_RESOLVED":
|
||||
return {"eligible": False, "reason_code": f"IPI_STATUS_{decision.get('ipi_status')}", "proposed_value": None}
|
||||
|
||||
ipi = (decision.get("classification") or {}).get("ipi")
|
||||
if not ipi:
|
||||
return {"eligible": False, "reason_code": "NO_IPI_DECISION", "proposed_value": None}
|
||||
|
||||
rate_kind = ipi.get("rate_kind")
|
||||
if rate_kind != "NUMERIC":
|
||||
return {"eligible": False, "reason_code": f"RATE_KIND_{rate_kind}", "proposed_value": None}
|
||||
|
||||
proposed_value = ipi.get("rate_numeric")
|
||||
if proposed_value is None:
|
||||
return {"eligible": False, "reason_code": "NO_NUMERIC_VALUE", "proposed_value": None}
|
||||
|
||||
if current_ipi is not None and str(current_ipi).strip() == str(proposed_value).strip():
|
||||
return {"eligible": False, "reason_code": "SAME_VALUE", "proposed_value": proposed_value}
|
||||
|
||||
return {"eligible": True, "reason_code": "CHANGE_REQUIRED", "proposed_value": proposed_value}
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Shared per-product IPI auto-apply pipeline: Hub decision -> dry-run ->
|
||||
guarded write -> local SQLite audit. Used by both the manual CLI
|
||||
(run_agent_auto_apply.py) and the long-running job-polling service
|
||||
(run_agent_service.py) so the processing logic exists in exactly one place.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .ipi_auto_apply import ipi_write_decision
|
||||
from .runner import product_fingerprint
|
||||
from central.app.operational_state import classify_product_state, summarize_operational_states
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def run_pipeline(products, decisions, state, firebird_writes, write_connection, executor, database_path, authorized_path=None):
|
||||
"""Process every (product, decision) pair. Returns a summary dict.
|
||||
|
||||
products come from a fresh Firebird read (current IPI is always "known").
|
||||
decisions come from the Hub's /v1/classify/batch, one per product, same order.
|
||||
"""
|
||||
summary = {
|
||||
"processed": 0,
|
||||
"auto_apply_eligible": 0,
|
||||
"writes_attempted": 0,
|
||||
"writes_succeeded": 0,
|
||||
"writes_failed": 0,
|
||||
"skipped_not_eligible": 0,
|
||||
"no_ops": 0,
|
||||
"product_states": [],
|
||||
"blocked_products": [],
|
||||
"resolved_products": [],
|
||||
}
|
||||
for product, decision in zip(products, decisions):
|
||||
code = product["CODIGO"]
|
||||
fingerprint = product_fingerprint(product)
|
||||
current_ipi = product.get("IPI")
|
||||
summary["processed"] += 1
|
||||
|
||||
wd = ipi_write_decision(decision, current_ipi)
|
||||
decision_status = decision.get("status")
|
||||
ipi_status = decision.get("ipi_status")
|
||||
operational = classify_product_state(product, decision)
|
||||
summary["product_states"].append(operational)
|
||||
if operational["state"].startswith("BLOCKED_"):
|
||||
summary["blocked_products"].append({
|
||||
"company": product.get("COMPANY") or product.get("company") or "",
|
||||
"product_code": code,
|
||||
"gtin": product.get("GTIN") or product.get("EAN_GTIN"),
|
||||
"description": product.get("PRODUTO") or product.get("description"),
|
||||
"blocker_type": operational["state"],
|
||||
"field": "NCM",
|
||||
"reason_code": decision_status or "MISSING_EVIDENCE",
|
||||
"evidence_summary": {"decision_status": decision_status, "ipi_status": ipi_status},
|
||||
"input_fingerprint": fingerprint,
|
||||
})
|
||||
else:
|
||||
summary["resolved_products"].append(code)
|
||||
|
||||
if not firebird_writes:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "DRY_RUN_ONLY", current_ipi, wd["proposed_value"], None, wd["reason_code"], now())
|
||||
if wd["eligible"]:
|
||||
summary["auto_apply_eligible"] += 1
|
||||
elif wd["reason_code"] == "SAME_VALUE":
|
||||
summary["no_ops"] += 1
|
||||
continue
|
||||
|
||||
if not wd["eligible"]:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "SKIPPED_NOT_ELIGIBLE", current_ipi, wd["proposed_value"], None, wd["reason_code"], now())
|
||||
summary["skipped_not_eligible"] += 1
|
||||
if wd["reason_code"] == "SAME_VALUE":
|
||||
summary["no_ops"] += 1
|
||||
continue
|
||||
|
||||
summary["auto_apply_eligible"] += 1
|
||||
summary["writes_attempted"] += 1
|
||||
proposal = {
|
||||
"product_code": code,
|
||||
"field": "IPI",
|
||||
"current_value": current_ipi,
|
||||
"proposed_value": wd["proposed_value"],
|
||||
"status": "CHANGE_REQUIRED",
|
||||
"write_eligible": True,
|
||||
"source_dataset": "TIPI_IPI",
|
||||
"source_version": decision.get("dataset_versions", {}).get("tipi_ipi"),
|
||||
}
|
||||
try:
|
||||
record = executor.execute_guarded_update(write_connection, database_path=database_path, proposal=proposal, now=now(), authorized_path=authorized_path)
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "SUCCEEDED", record.before_value, record.after_value, f"rowcount={record.rowcount}", wd["reason_code"], now())
|
||||
summary["writes_succeeded"] += 1
|
||||
summary["product_states"][-1] = classify_product_state(product, decision, write_status="SUCCEEDED")
|
||||
except executor.WriteRefused as error:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "FAILED", current_ipi, wd["proposed_value"], str(error), wd["reason_code"], now())
|
||||
summary["writes_failed"] += 1
|
||||
summary["product_states"][-1] = classify_product_state(product, decision, write_status="FAILED")
|
||||
|
||||
summary["operational"] = summarize_operational_states(summary["product_states"])
|
||||
return summary
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Single installation preflight for the GTI Fiscal Agent. READ-ONLY: never writes to Firebird, never prints secrets.
|
||||
|
||||
PREFLIGHT=PASS | FAIL (+ blockers, one line each)
|
||||
|
||||
Checks: config, FISCAL_MODE/FIREBIRD_WRITES, Hub reachable/auth/version, Firebird reachable + version + ODS, C000004/C000025,
|
||||
capabilities (minimum: NCM, IPI, CEST), company context (UF/CRT, deterministic resolver), schema/type guards, SQLite state, log dir.
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_VERSION_DEFAULT = "1.3.9"
|
||||
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
|
||||
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
|
||||
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
|
||||
|
||||
|
||||
def _check(name, ok, detail="", warn=False):
|
||||
return {"name": name, "status": "PASS" if ok else ("WARN" if warn else "FAIL"), "detail": detail}
|
||||
|
||||
|
||||
def major_minor(version):
|
||||
parts = str(version or "").replace("-", ".").replace("rc", ".").split(".")
|
||||
return ".".join(parts[:2])
|
||||
|
||||
|
||||
def default_http_get(url, token=None, timeout=10):
|
||||
import httpx
|
||||
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
||||
r = httpx.get(url, headers=headers, timeout=timeout)
|
||||
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
|
||||
|
||||
|
||||
def firebird_probe(env):
|
||||
"""Read-only introspection of the configured database."""
|
||||
import fdb
|
||||
from .company_context import collect_evidence, config_from_env, resolve
|
||||
conn = fdb.connect(database=env["FIREBIRD_DATABASE_PATH"], user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH)
|
||||
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE WHERE rf.RDB$RELATION_NAME IN ('C000004', 'C000025')""")
|
||||
tables = {}
|
||||
for t, c, ty, ln in cur.fetchall():
|
||||
tables.setdefault(t, {})[c] = (ty, ln)
|
||||
products = None
|
||||
if "C000025" in tables:
|
||||
cur.execute("SELECT COUNT(*) FROM C000025")
|
||||
products = cur.fetchone()[0]
|
||||
company = resolve(collect_evidence(cur), config_from_env(env))
|
||||
return {"server_version": getattr(conn, "server_version", None), "ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}",
|
||||
"tables": tables, "products": products, "company": company}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def run_preflight(env=None, *, http_get=default_http_get, probe=firebird_probe, agent_version=None):
|
||||
from .capabilities import detect_capabilities, type_ok
|
||||
from .company_context import preflight_report
|
||||
env = env if env is not None else os.environ
|
||||
checks, info = [], {}
|
||||
agent_version = agent_version or env.get("AGENT_VERSION") or AGENT_VERSION_DEFAULT
|
||||
|
||||
missing = [k for k in REQUIRED_ENV if not (env.get(k) or "").strip()]
|
||||
checks.append(_check("CONFIG", not missing, f"missing: {', '.join(missing)}" if missing else "required variables present"))
|
||||
|
||||
mode = (env.get("FISCAL_MODE") or "").strip().upper()
|
||||
writes = (env.get("FIREBIRD_WRITES") or "0").strip() == "1"
|
||||
checks.append(_check("FISCAL_MODE", mode in FISCAL_MODES, f"FISCAL_MODE={mode or 'unset'} (must be one of {FISCAL_MODES})"))
|
||||
dry_conflict = mode == "DRY_RUN" and writes
|
||||
checks.append(_check("FIREBIRD_WRITES", not dry_conflict, "FIREBIRD_WRITES=1 conflicts with FISCAL_MODE=DRY_RUN (writes stay disabled; fix the config)" if dry_conflict else f"requested={int(writes)}"))
|
||||
effective = writes and mode == "AUTO_APPLY"
|
||||
info["effective_firebird_writes"] = effective
|
||||
if effective:
|
||||
ok = bool((env.get("AUTHORIZED_WRITE_DATABASE_PATH") or "").strip() and (env.get("EXPECTED_SCHEMA_FINGERPRINT") or "").strip())
|
||||
checks.append(_check("WRITE_AUTHORIZATION", ok, "AUTO_APPLY needs AUTHORIZED_WRITE_DATABASE_PATH and EXPECTED_SCHEMA_FINGERPRINT (two independent layers)"))
|
||||
|
||||
hub = (env.get("GTI_HUB_URL") or "").rstrip("/")
|
||||
hub_ok = False
|
||||
if hub:
|
||||
try:
|
||||
code, _ = http_get(f"{hub}/health")
|
||||
hub_ok = code == 200
|
||||
checks.append(_check("HUB_REACHABLE", hub_ok, f"GET /health -> {code}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("HUB_REACHABLE", False, type(e).__name__))
|
||||
if hub_ok:
|
||||
try:
|
||||
code, body = http_get(f"{hub}/version")
|
||||
hub_v = body.get("version")
|
||||
info["hub_version"] = hub_v
|
||||
checks.append(_check("HUB_VERSION_COMPATIBLE", code == 200 and major_minor(hub_v) == major_minor(agent_version), f"hub={hub_v} agent={agent_version} (major.minor must match)"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("HUB_VERSION_COMPATIBLE", False, type(e).__name__))
|
||||
try:
|
||||
code, _ = http_get(f"{hub}/v1/datasets/status", token=env.get("AGENT_TOKEN"))
|
||||
checks.append(_check("AUTH", code == 200, f"agent token -> HTTP {code}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("AUTH", False, type(e).__name__))
|
||||
|
||||
fb = None
|
||||
if all((env.get(k) or "").strip() for k in ("FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")):
|
||||
try:
|
||||
fb = probe(env)
|
||||
checks.append(_check("FIREBIRD_REACHABLE", True, "read-only connection opened"))
|
||||
info.update(firebird_version=fb.get("server_version"), ods=fb.get("ods"), products=fb.get("products"))
|
||||
checks.append(_check("FIREBIRD_VERSION_ODS", bool(fb.get("server_version")), f"{fb.get('server_version')} / ODS {fb.get('ods')}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("FIREBIRD_REACHABLE", False, f"{type(e).__name__}: {str(e)[:80]}"))
|
||||
if fb:
|
||||
t = fb["tables"]
|
||||
checks.append(_check("C000004", "C000004" in t, "company table present" if "C000004" in t else "missing"))
|
||||
checks.append(_check("C000025", "C000025" in t, "product table present" if "C000025" in t else "missing"))
|
||||
cols = {c: ty for c, (ty, _) in t.get("C000025", {}).items()}
|
||||
caps = detect_capabilities(cols)
|
||||
info["capabilities"] = caps
|
||||
lacking = [c for c in REQUIRED_CAPABILITIES if not caps.get(c)]
|
||||
checks.append(_check("CAPABILITIES", not lacking, f"missing required: {', '.join(lacking)}" if lacking else f"required present; optional true: {sum(1 for c, v in caps.items() if v and c not in REQUIRED_CAPABILITIES)}"))
|
||||
rep = preflight_report(fb["company"])
|
||||
info["company_context"] = rep
|
||||
checks.append(_check("COMPANY_CONTEXT", rep["COMPANY_CONTEXT"] == "PASS", "; ".join(rep.get("reasons") or []) or f"{rep['classification']} source={rep['source_of_truth']}"))
|
||||
checks.append(_check("UF", bool(rep.get("company_uf")), str(rep.get("company_uf"))))
|
||||
checks.append(_check("CRT", bool(rep.get("company_crt")), str(rep.get("company_crt"))))
|
||||
writable = [f for f in ("IPI", "CEST", "EX_TIPI", "CLASSIFICACAO_FISCAL") if f in t.get("C000025", {})]
|
||||
bad = [f for f in writable if not type_ok(f, t["C000025"][f][0])]
|
||||
lens = {f: t["C000025"][f][1] for f in ("CEST", "EX_TIPI") if f in t.get("C000025", {})}
|
||||
checks.append(_check("SCHEMA_TYPE_GUARDS", not bad and len(writable) >= 3, f"incompatible: {bad}" if bad else f"types ok for {writable}; text lengths {lens} (the executor re-checks length/type before any write)"))
|
||||
|
||||
state_db = env.get("AGENT_STATE_DB") or str(Path.cwd() / "agent_state.sqlite")
|
||||
try:
|
||||
Path(state_db).parent.mkdir(parents=True, exist_ok=True)
|
||||
c = sqlite3.connect(state_db)
|
||||
c.execute("SELECT 1")
|
||||
c.close()
|
||||
checks.append(_check("SQLITE", True, "state database opens"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("SQLITE", False, type(e).__name__))
|
||||
log_path = Path(env.get("AGENT_LOG_PATH") or Path.cwd() / "logs" / "agent.log")
|
||||
try:
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
checks.append(_check("LOG_DIRECTORY", os.access(log_path.parent, os.W_OK), "log directory writable"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("LOG_DIRECTORY", False, type(e).__name__))
|
||||
|
||||
blockers = [f"{c['name']}: {c['detail']}" for c in checks if c["status"] == "FAIL"]
|
||||
return {"PREFLIGHT": "FAIL" if blockers else "PASS", "blockers": blockers, "checks": checks, "agent_version": agent_version, "fiscal_mode": mode or None, **info}
|
||||
@@ -0,0 +1,214 @@
|
||||
"""RTC auto-apply: fills C000025.CCLASSTRIB, and NOTHING else, when a deterministic rule proves the class.
|
||||
|
||||
Safety model (defense in depth, every gate must hold, default is OFF): the Hub sets a per-installation mode DISABLED / DRY_RUN / APPROVAL_REQUIRED / ENABLED and
|
||||
delivers it inside an authenticated job. The Agent only exposes the restricted capability RTC_CCLASSTRIB_WRITE_V1; there is no local switch to edit (no agent.env, no FIREBIRD_WRITES).
|
||||
Write allowlist: ALLOWED_RTC_WRITE_FIELDS = ["CCLASSTRIB"] on C000025 (no free SQL exists). One short transaction per product. Immediately before the UPDATE the product,
|
||||
company, RTC parameter and local catalog are re-read and the whole decision is re-derived; if ANYTHING changed that product is ABORTED and nothing is written.
|
||||
An existing value is never overwritten. Every applied change returns an audit record with the old value; ROLLBACK restores it only if the value still equals what was written.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
from central.app.cclasstrib_resolver import RULE_VERSION, compare_catalogs, local_catalog_hash, resolve_cclasstrib
|
||||
from central.app.unified_fiscal_coverage import rtc_enabled
|
||||
|
||||
from .unified_fiscal import detect_rtc_capabilities, load_rtc_context
|
||||
|
||||
ALLOWED_RTC_WRITE_FIELDS = ("CCLASSTRIB",)
|
||||
DEFAULT_MAX_PER_JOB = 500
|
||||
CAPABILITY = "RTC_CCLASSTRIB_WRITE_V1"
|
||||
MODES = ("DISABLED", "DRY_RUN", "APPROVAL_REQUIRED", "ENABLED")
|
||||
MAX_PREVIEW_CANDIDATES = 5000
|
||||
|
||||
|
||||
class RtcApplyAborted(Exception):
|
||||
"""The product is left untouched; `reason` is reported in the audit."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def _digits(value) -> str:
|
||||
return re.sub(r"\D", "", _text(value))
|
||||
|
||||
|
||||
def write_gate(*, mode: str, capability: bool = True, approved: bool = True) -> dict:
|
||||
"""{"allowed": bool, "missing": [...]}: a real write needs the capability, an authenticated Hub mode that permits writing (ENABLED, or APPROVAL_REQUIRED for approved products)."""
|
||||
mode = mode if mode in MODES else "DISABLED"
|
||||
missing = []
|
||||
if not capability:
|
||||
missing.append("CAPABILITY")
|
||||
if mode not in ("ENABLED", "APPROVAL_REQUIRED"):
|
||||
missing.append("HUB_MODE_" + mode)
|
||||
elif mode == "APPROVAL_REQUIRED" and not approved:
|
||||
missing.append("HUB_APPROVAL")
|
||||
return {"allowed": not missing, "missing": missing, "mode": mode, "capability": bool(capability)}
|
||||
|
||||
|
||||
def candidate_fingerprint(code, ncm, description, current) -> str:
|
||||
import hashlib
|
||||
return hashlib.sha256("|".join((_text(code), _digits(ncm), _text(description), _text(current))).encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def select_candidates(findings: dict, products: list[dict], *, limit: int = DEFAULT_MAX_PER_JOB) -> tuple[list[dict], int]:
|
||||
"""Products READY_TO_APPLY: empty CCLASSTRIB + a deterministic resolution, no other RTC problem. Returns (candidates up to `limit`, total ready)."""
|
||||
by_code = {str(p.get("CODIGO") or ""): p for p in products}
|
||||
ready = []
|
||||
for code, found in findings.items():
|
||||
res = found.get("rtc_resolution") or {}
|
||||
if not res.get("auto_apply_candidate"):
|
||||
continue
|
||||
rtc_types = {b["type"] for b in found.get("blockers", []) if b["category"] == "RTC"}
|
||||
if rtc_types != {"CCLASSTRIB_MISSING"}:
|
||||
continue
|
||||
p = by_code.get(code, {})
|
||||
if _text(p.get("CCLASSTRIB")):
|
||||
continue
|
||||
ready.append({"product_code": code, "proposed": res["resolved_cclasstrib"], "cst": res["resolved_cst_ibscbs"], "rule_id": res["rule_id"], "ncm": _digits(p.get("CLASSIFICACAO_FISCAL")), "confidence_type": res["confidence_type"], "description": _text(p.get("PRODUTO"))[:255], "current": "", "basis": f'{res["rule_id"]}:{res["confidence_type"]}',
|
||||
"fingerprint": candidate_fingerprint(code, p.get("CLASSIFICACAO_FISCAL"), p.get("PRODUTO"), "")})
|
||||
return ready[:limit], len(ready)
|
||||
|
||||
|
||||
def apply_candidate(conn, candidate: dict, *, company: dict, decision: dict, now: str, job_id=None, today: date | None = None) -> dict:
|
||||
"""One product, one short transaction. `decision` = {"local_catalog_hash": ...} captured when the candidate was chosen. Raises RtcApplyAborted (nothing written)."""
|
||||
field = "CCLASSTRIB"
|
||||
assert field in ALLOWED_RTC_WRITE_FIELDS
|
||||
today = today or date.today()
|
||||
code, proposed = candidate["product_code"], candidate["proposed"]
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
schema, catalog, extras = load_rtc_context(cur) # fresh: the ERP is asked again, nothing cached
|
||||
caps = detect_rtc_capabilities(schema, param_value=extras["param_value"], catalog=catalog, physical_fk=extras["physical_fk"])
|
||||
if extras["param_conflict"] or not rtc_enabled(caps):
|
||||
raise RtcApplyAborted("RTC_NO_LONGER_ENABLED")
|
||||
if not caps["RTC_CATALOG_VALID"]:
|
||||
raise RtcApplyAborted("RTC_CATALOG_INVALID")
|
||||
local_rows = (catalog or {}).get("rows") or {}
|
||||
fresh_hash = local_catalog_hash(local_rows)
|
||||
if decision.get("local_catalog_hash") and fresh_hash != decision["local_catalog_hash"]:
|
||||
raise RtcApplyAborted("LOCAL_CATALOG_CHANGED_SINCE_DECISION")
|
||||
cur.execute("SELECT DISTINCT TRIM(CRT) FROM C000004")
|
||||
crts = {_text(r[0]) for r in cur.fetchall()}
|
||||
expected_crt = _text((company or {}).get("crt"))
|
||||
if not expected_crt or crts != {expected_crt}:
|
||||
raise RtcApplyAborted("COMPANY_CONTEXT_CHANGED")
|
||||
if expected_crt != "3":
|
||||
raise RtcApplyAborted("REGIME_NOT_NORMAL")
|
||||
cur.execute(f"SELECT {field}, CLASSIFICACAO_FISCAL, PRODUTO FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
raise RtcApplyAborted("PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND")
|
||||
current, ncm = _text(rows[0][0]), _digits(rows[0][1])
|
||||
if current:
|
||||
raise RtcApplyAborted("VALUE_ALREADY_SET_NEVER_OVERWRITTEN" if current != proposed else "NO_WRITE_REQUIRED")
|
||||
if ncm != candidate["ncm"]:
|
||||
raise RtcApplyAborted("PRODUCT_CHANGED_NCM")
|
||||
if candidate.get("fingerprint") and candidate_fingerprint(code, ncm, rows[0][2], current) != candidate["fingerprint"]:
|
||||
raise RtcApplyAborted("PRODUCT_CHANGED_SINCE_DECISION")
|
||||
compat = compare_catalogs(local_rows, today)
|
||||
resolution = resolve_cclasstrib({"CODIGO": code, "CLASSIFICACAO_FISCAL": ncm, "ncm_valid": True}, company, {"rtc_enabled": True, "local_catalog_valid": True}, local_rows, compat=compat, today=today)
|
||||
if not resolution.auto_apply_candidate or resolution.resolved_cclasstrib != proposed or resolution.rule_id != candidate["rule_id"]:
|
||||
raise RtcApplyAborted("DECISION_CHANGED")
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (proposed, code))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
raise RtcApplyAborted("UPDATE_ROWCOUNT_NOT_ONE")
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = _text(cur.fetchone()[0])
|
||||
conn.commit()
|
||||
if after != proposed:
|
||||
raise RuntimeError(f"post-write verification failed for {code}: {after!r} != {proposed!r}")
|
||||
official = compat["official_catalog"]
|
||||
return {"outcome": "APPLIED", "product_code": code, "field": field, "old_value": None, "new_value": proposed, "cst_ibscbs": resolution.resolved_cst_ibscbs, "rule_id": resolution.rule_id,
|
||||
"confidence_type": resolution.confidence_type, "evidence": {"inputs_used": resolution.inputs_used, "evidence": resolution.evidence[:5], "reason": resolution.reason},
|
||||
"official_source_version": {"cclasstrib_table": official["publication"], "cclasstrib_sha256": official["sha256"], "lc214_sha256": resolution.source_references[0]["sha256"]},
|
||||
"local_catalog_hash": fresh_hash, "timestamp": now, "job_id": job_id, "rowcount": 1, "verified_after": True, "action": "APPLY", "rule_version": RULE_VERSION,
|
||||
"company_crt": expected_crt, "description_snapshot": _text(rows[0][2])[:255], "rtc_status": "ENABLED", "local_catalog_state": (compat.get("per_code") or {}).get(proposed), "authorization_origin": decision.get("origin")}
|
||||
except RtcApplyAborted:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
raise
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def apply_candidates(conn, candidates: list[dict], *, company: dict, decision: dict, now: str, job_id=None, today: date | None = None, authorize=None) -> dict:
|
||||
"""Applies each candidate independently; a failure or an abort never stops the rest and never leaves a partial change.
|
||||
`authorize(product_code)` (optional) re-asks the Hub, right before each UPDATE, whether the central authorization still permits it: it returns None to proceed or a
|
||||
reason (AUTHORIZATION_CHANGED / AUTHORIZATION_UNVERIFIABLE) to skip the product with zero writes."""
|
||||
applied, aborted, reasons = [], [], {}
|
||||
for candidate in candidates:
|
||||
if authorize is not None:
|
||||
denied = authorize(candidate["product_code"])
|
||||
if denied:
|
||||
aborted.append({"outcome": "ABORTED", "product_code": candidate["product_code"], "field": "CCLASSTRIB", "reason": denied, "rule_id": candidate["rule_id"], "job_id": job_id, "timestamp": now, "action": "APPLY", "authorization_origin": decision.get("origin")})
|
||||
reasons[denied] = reasons.get(denied, 0) + 1
|
||||
continue
|
||||
try:
|
||||
applied.append(apply_candidate(conn, candidate, company=company, decision=decision, now=now, job_id=job_id, today=today))
|
||||
except RtcApplyAborted as stop:
|
||||
aborted.append({"outcome": "ABORTED", "product_code": candidate["product_code"], "field": "CCLASSTRIB", "reason": stop.reason, "rule_id": candidate["rule_id"], "job_id": job_id, "timestamp": now, "action": "APPLY", "authorization_origin": decision.get("origin")})
|
||||
reasons[stop.reason] = reasons.get(stop.reason, 0) + 1
|
||||
except Exception as error: # noqa: BLE001 - technical failure of ONE product must not abort the batch
|
||||
aborted.append({"outcome": "FAILED", "product_code": candidate["product_code"], "field": "CCLASSTRIB", "reason": f"{type(error).__name__}", "rule_id": candidate["rule_id"], "job_id": job_id, "timestamp": now})
|
||||
reasons["TECHNICAL_FAILURE"] = reasons.get("TECHNICAL_FAILURE", 0) + 1
|
||||
return {"applied": applied, "aborted": aborted, "aborted_reasons": reasons, "write_attempts": len(candidates), "write_successes": len(applied)}
|
||||
|
||||
|
||||
def rollback_one(conn, item: dict, *, now: str, job_id=None) -> dict:
|
||||
"""Restores C000025.CCLASSTRIB to empty for ONE previously applied audit record, only if the value still equals what was written. No free SQL: the only statement is fixed."""
|
||||
field = "CCLASSTRIB"
|
||||
assert field in ALLOWED_RTC_WRITE_FIELDS
|
||||
code, written = _text(item.get("product_code")), _text(item.get("written_value"))
|
||||
base = {"product_code": code, "field": field, "job_id": job_id, "timestamp": now, "action": "ROLLBACK", "rollback_of": item.get("audit_id"), "old_value": written, "new_value": None, "rule_id": item.get("rule_id")}
|
||||
if not code or not written or item.get("restore_value"):
|
||||
return {**base, "outcome": "ABORTED", "reason": "ROLLBACK_REQUEST_INVALID"}
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
|
||||
current = _text(rows[0][0])
|
||||
if current != written:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "ROLLBACK_CONFLICT_VALUE_CHANGED" if current else "ALREADY_EMPTY"}
|
||||
cur.execute(f"UPDATE C000025 SET {field} = NULL WHERE CODIGO = ? AND TRIM({field}) = ?", (code, written))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"}
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = _text(cur.fetchone()[0])
|
||||
conn.commit()
|
||||
if after:
|
||||
raise RuntimeError("post-rollback verification failed")
|
||||
return {**base, "outcome": "ROLLED_BACK", "rowcount": 1, "verified_after": True}
|
||||
except Exception as error: # noqa: BLE001
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return {**base, "outcome": "FAILED", "reason": type(error).__name__}
|
||||
|
||||
|
||||
def rollback_records(conn, items: list[dict], *, now: str, job_id=None) -> dict:
|
||||
results = [rollback_one(conn, item, now=now, job_id=job_id) for item in items]
|
||||
return {"rollbacks": results, "rolled_back": sum(1 for r in results if r["outcome"] == "ROLLED_BACK")}
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Gate Firebird reads on the registered schema fingerprint."""
|
||||
import hashlib, json
|
||||
REFERENCE_SCHEMA_V1 = "1198f89897532b643439f06b729e93b189018f56e7d478e95095f8f56977005b"
|
||||
|
||||
def product_fingerprint(product):
|
||||
fields = ("PRODUTO", "EAN_GTIN", "NCM", "CLASSIFICACAO_FISCAL", "CFOP", "CST", "ALIQUOTA", "CEST", "CSOSN")
|
||||
return hashlib.sha256(json.dumps({key: product.get(key) for key in fields}, sort_keys=True, default=str).encode()).hexdigest()
|
||||
def to_hub_product(product):
|
||||
# C000025 has NCM empty and stores the confirmed eight-digit NCM in CLASSIFICACAO_FISCAL.
|
||||
return {"product_id": product["CODIGO"], "description": product.get("PRODUTO", ""), "gtin": product.get("EAN_GTIN"), "current_ncm": product.get("CLASSIFICACAO_FISCAL")}
|
||||
def run_sample(products, client, state, schema_fingerprint):
|
||||
if schema_fingerprint != REFERENCE_SCHEMA_V1: return {"status": "SCHEMA_UNKNOWN", "results": [], "firebird_writes": 0}
|
||||
outbound = []
|
||||
for product in products:
|
||||
fingerprint = product_fingerprint(product)
|
||||
if state.enqueue(product["CODIGO"], fingerprint, "pending", "0.1.0"): outbound.append((product, fingerprint))
|
||||
if not outbound: return {"status": "OK", "results": [], "firebird_writes": 0, "api_requests": 0}
|
||||
try: results = client.classify_batch([to_hub_product(product) for product, _ in outbound])
|
||||
except Exception:
|
||||
for product, _ in outbound: state.retry(product["CODIGO"])
|
||||
return {"status": "API_UNAVAILABLE", "results": [], "firebird_writes": 0}
|
||||
for (product, fingerprint), result in zip(outbound, results): state.result_received(product["CODIGO"], fingerprint, result["dataset_versions"].get("ncm") or "none", result["engine_version"])
|
||||
return {"status": "OK", "results": results, "firebird_writes": 0, "api_requests": 1}
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Local SQLite outbox; it has no Firebird dependency or write path."""
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
|
||||
class AgentState:
|
||||
def __init__(self, path):
|
||||
self.connection = sqlite3.connect(Path(path)); self.connection.row_factory = sqlite3.Row
|
||||
self.connection.executescript("CREATE TABLE IF NOT EXISTS product_state(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, dataset_version TEXT NOT NULL, engine_version TEXT NOT NULL, status TEXT NOT NULL, last_processed_at TEXT, last_result_hash TEXT, reason_code TEXT); CREATE TABLE IF NOT EXISTS outbox(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, status TEXT NOT NULL, retries INTEGER NOT NULL DEFAULT 0); CREATE TABLE IF NOT EXISTS ipi_write_log(product_id TEXT NOT NULL, source_fingerprint TEXT NOT NULL, decision_status TEXT, ipi_status TEXT, write_status TEXT NOT NULL, before_value TEXT, proposed_value TEXT, result TEXT, reason_code TEXT, timestamp TEXT NOT NULL);")
|
||||
self.connection.execute("CREATE TABLE IF NOT EXISTS installation_identity (id INTEGER PRIMARY KEY CHECK(id=1), installation_id TEXT NOT NULL)")
|
||||
# Last Hub decision per product + the source fingerprint it was computed for: what makes
|
||||
# SYNC_INCREMENTAL a real delta (only new/changed products are re-classified).
|
||||
self.connection.execute("CREATE TABLE IF NOT EXISTS decision_cache (product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, decision_json TEXT NOT NULL)")
|
||||
self.connection.commit()
|
||||
def stored_installation_id(self):
|
||||
row = self.connection.execute("SELECT installation_id FROM installation_identity WHERE id=1").fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
try:
|
||||
return str(uuid.UUID(row["installation_id"]))
|
||||
except (TypeError, ValueError, AttributeError) as error:
|
||||
raise ValueError("stored installation_id is not a valid UUID") from error
|
||||
def installation_id(self, candidate=None):
|
||||
existing = self.stored_installation_id()
|
||||
if existing:
|
||||
if candidate is not None and str(uuid.UUID(str(candidate))) != existing:
|
||||
raise ValueError("configured installation_id conflicts with persisted identity")
|
||||
return existing
|
||||
if candidate is None or not str(candidate).strip():
|
||||
raise ValueError("installation_id is required; refusing to generate a new identity")
|
||||
try:
|
||||
value = str(uuid.UUID(str(candidate)))
|
||||
except (TypeError, ValueError, AttributeError) as error:
|
||||
raise ValueError("installation_id must be a valid UUID") from error
|
||||
self.connection.execute("INSERT OR IGNORE INTO installation_identity(id, installation_id) VALUES(1, ?)", (value,))
|
||||
self.connection.commit()
|
||||
row = self.connection.execute("SELECT installation_id FROM installation_identity WHERE id=1").fetchone()
|
||||
if row is None or str(uuid.UUID(row["installation_id"])) != value:
|
||||
raise ValueError("persisted installation_id conflicts with candidate")
|
||||
return value
|
||||
def cached_fingerprint(self, product_id):
|
||||
row = self.connection.execute("SELECT source_fingerprint FROM decision_cache WHERE product_id=?", (product_id,)).fetchone()
|
||||
return row["source_fingerprint"] if row else None
|
||||
def cached_decision(self, product_id):
|
||||
row = self.connection.execute("SELECT decision_json FROM decision_cache WHERE product_id=?", (product_id,)).fetchone()
|
||||
return json.loads(row["decision_json"]) if row else None
|
||||
def remember_decision(self, product_id, source_fingerprint, decision):
|
||||
self.connection.execute("INSERT OR REPLACE INTO decision_cache(product_id,source_fingerprint,decision_json) VALUES(?,?,?)", (product_id, source_fingerprint, json.dumps(decision, default=str)))
|
||||
self.connection.commit()
|
||||
def forget_missing(self, present_product_ids):
|
||||
"""Products no longer in the ERP leave the LOCAL cache only; Hub-side history/blockers are never deleted by this."""
|
||||
present = set(present_product_ids)
|
||||
gone = [row["product_id"] for row in self.connection.execute("SELECT product_id FROM decision_cache") if row["product_id"] not in present]
|
||||
for product_id in gone:
|
||||
self.connection.execute("DELETE FROM decision_cache WHERE product_id=?", (product_id,))
|
||||
self.connection.commit()
|
||||
return len(gone)
|
||||
def enqueue(self, product_id, source_fingerprint, dataset_version, engine_version):
|
||||
row = self.connection.execute("SELECT * FROM product_state WHERE product_id=?", (product_id,)).fetchone()
|
||||
if row and tuple(row[key] for key in ("source_fingerprint", "engine_version", "status")) == (source_fingerprint, engine_version, "RESULT_RECEIVED"): return False
|
||||
self.connection.execute("INSERT OR REPLACE INTO product_state(product_id,source_fingerprint,dataset_version,engine_version,status) VALUES(?,?,?,?,?)", (product_id, source_fingerprint, dataset_version, engine_version, "PENDING_API")); self.connection.execute("INSERT OR REPLACE INTO outbox(product_id,source_fingerprint,status) VALUES(?,?,?)", (product_id, source_fingerprint, "PENDING_API")); self.connection.commit(); return True
|
||||
def pending(self): return [dict(row) for row in self.connection.execute("SELECT * FROM outbox WHERE status='PENDING_API'")]
|
||||
def retry(self, product_id): self.connection.execute("UPDATE outbox SET retries=retries+1,status='PENDING_API' WHERE product_id=?", (product_id,)); self.connection.commit()
|
||||
def result_received(self, product_id, source_fingerprint, dataset_version, engine_version):
|
||||
self.connection.execute("UPDATE product_state SET source_fingerprint=?,dataset_version=?,engine_version=?,status='RESULT_RECEIVED' WHERE product_id=?", (source_fingerprint, dataset_version, engine_version, product_id)); self.connection.execute("DELETE FROM outbox WHERE product_id=?", (product_id,)); self.connection.commit()
|
||||
def log_ipi_write(self, product_id, source_fingerprint, decision_status, ipi_status, write_status, before_value, proposed_value, result, reason_code, timestamp):
|
||||
self.connection.execute(
|
||||
"INSERT INTO ipi_write_log(product_id,source_fingerprint,decision_status,ipi_status,write_status,before_value,proposed_value,result,reason_code,timestamp) VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||||
(product_id, source_fingerprint, decision_status, ipi_status, write_status, str(before_value) if before_value is not None else None, str(proposed_value) if proposed_value is not None else None, result, reason_code, timestamp),
|
||||
)
|
||||
self.connection.commit()
|
||||
def ipi_write_log_for(self, product_id):
|
||||
return [dict(row) for row in self.connection.execute("SELECT * FROM ipi_write_log WHERE product_id=? ORDER BY timestamp", (product_id,))]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""READ-ONLY ERP loader for the unified fiscal completeness (PIS/COFINS applicability + RTC when the ERP has the reform on).
|
||||
|
||||
The rules live in central/app/unified_fiscal_coverage.py (pure). This module only observes the ERP through an already open
|
||||
READ ONLY connection (SELECT only): schema capabilities, RTC catalog, PARAMETROS.USAR_REFORMA_TRIBUTARIA and the product columns.
|
||||
Nothing is written and no value is ever filled in.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from central.app.cclasstrib_resolver import compare_catalogs, local_catalog_hash, resolve_cclasstrib
|
||||
from central.app.rtc_findings import refine_rtc_finding, summarize_rtc
|
||||
from central.app.unified_fiscal_coverage import PIS_COFINS_FIELDS, catalog_integrity, cclasstrib_applicability, evaluate_product, legal_effective_context, rtc_enabled
|
||||
|
||||
PRODUCT_TABLE = "C000025"
|
||||
RTC_CCLASSTRIB_TABLE = "RTC_CCLASSTRIB"
|
||||
RTC_CST_TABLE = "RTC_CST"
|
||||
PRODUCT_COLUMNS = ("CODIGO", "CCLASSTRIB", "CLASSIFICACAO_FISCAL") + PIS_COFINS_FIELDS + ("PISCOFINS", "INCIDENCIA_PISCOFINS")
|
||||
TRUTHY = {"1", "S", "T", "TRUE"}
|
||||
|
||||
|
||||
def detect_rtc_capabilities(schema: dict, *, param_value=None, catalog: dict | None = None, physical_fk: bool | None = None) -> dict:
|
||||
"""schema: {TABLE: {COLUMN, ...}}. Fail-closed: anything not proven is False. Old databases without RTC simply report False."""
|
||||
product_cols = set(schema.get(PRODUCT_TABLE, ()))
|
||||
class_cols = set(schema.get(RTC_CCLASSTRIB_TABLE, ()))
|
||||
cst_cols = set(schema.get(RTC_CST_TABLE, ()))
|
||||
has_product = "CCLASSTRIB" in product_cols
|
||||
has_class = {"CCLASSTRIB", "CSTIBSCBS"} <= class_cols
|
||||
has_cst = "CSTIBSCBS" in cst_cols
|
||||
caps = {
|
||||
"HAS_PRODUCT_CCLASSTRIB": has_product, "HAS_RTC_CCLASSTRIB_TABLE": has_class, "HAS_RTC_CST_TABLE": has_cst,
|
||||
"HAS_RTC_RELATIONSHIP": bool(has_product and has_class and has_cst), "HAS_PHYSICAL_RTC_FK": bool(physical_fk),
|
||||
"RTC_ENABLED_BY_PARAM": False if param_value is None else str(param_value).strip().upper() in TRUTHY,
|
||||
"RTC_CATALOG_VALID": False,
|
||||
}
|
||||
if catalog is not None and has_class and has_cst:
|
||||
caps["RTC_CATALOG_VALID"] = catalog_integrity(catalog)["valid"]
|
||||
return caps
|
||||
|
||||
|
||||
def load_rtc_context(cursor) -> tuple[dict, dict | None, dict]:
|
||||
"""Schema + catalog + ERP parameter through a READ ONLY cursor. Never raises for a database that lacks RTC."""
|
||||
schema: dict[str, set] = {}
|
||||
cursor.execute("SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS rf WHERE rf.RDB$RELATION_NAME IN ('C000025','RTC_CCLASSTRIB','RTC_CST','PARAMETROS')")
|
||||
for table, column in cursor.fetchall():
|
||||
schema.setdefault(table, set()).add(column)
|
||||
extras: dict = {"param_value": None, "param_rows": 0, "param_conflict": False, "physical_fk": False}
|
||||
if "USAR_REFORMA_TRIBUTARIA" in schema.get("PARAMETROS", ()):
|
||||
try:
|
||||
# ALL rows are read: the source of truth is what THIS installation's database says NOW (never a cached or Hub-side value).
|
||||
cursor.execute("SELECT FIRST 20 USAR_REFORMA_TRIBUTARIA FROM PARAMETROS")
|
||||
values = [None if r[0] is None else str(r[0]).strip().upper() for r in cursor.fetchall()]
|
||||
extras["param_rows"] = len(values)
|
||||
if len(set(values)) > 1:
|
||||
extras["param_conflict"] = True # rows disagree: never guess "on" (RTC stays out of scope, and the conflict is reported)
|
||||
elif values:
|
||||
extras["param_value"] = values[0]
|
||||
except Exception: # noqa: BLE001 - unreadable parameter => RTC stays off (not enabled by the ERP)
|
||||
extras["param_value"] = None
|
||||
try:
|
||||
cursor.execute("SELECT COUNT(*) FROM RDB$RELATION_CONSTRAINTS WHERE RDB$RELATION_NAME = 'C000025' AND RDB$CONSTRAINT_TYPE = 'FOREIGN KEY' AND RDB$CONSTRAINT_NAME CONTAINING 'CCLASSTRIB'")
|
||||
extras["physical_fk"] = cursor.fetchone()[0] > 0
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
catalog = None
|
||||
class_cols = schema.get(RTC_CCLASSTRIB_TABLE, set())
|
||||
if {"CCLASSTRIB", "CSTIBSCBS"} <= class_cols and "CSTIBSCBS" in schema.get(RTC_CST_TABLE, set()):
|
||||
validity = "INICIO_VIGENCIA" in class_cols and "FIM_VIGENCIA" in class_cols
|
||||
extra = [c for c in ("TIPO_ALIQUOTA", "PRED_IBS", "PRED_CBS", "IND_NFE", "IND_NFCE") if c in class_cols]
|
||||
cursor.execute("SELECT TRIM(CCLASSTRIB), TRIM(CSTIBSCBS)" + (", INICIO_VIGENCIA, FIM_VIGENCIA" if validity else "") + "".join(f", {c}" for c in extra) + " FROM RTC_CCLASSTRIB")
|
||||
rows = cursor.fetchall()
|
||||
classes = {r[0]: (r[1] or None) for r in rows}
|
||||
cursor.execute("SELECT TRIM(CSTIBSCBS) FROM RTC_CST")
|
||||
catalog = {"classes": classes, "cst": {r[0] for r in cursor.fetchall()}, "duplicates": len(rows) - len(classes)}
|
||||
if validity:
|
||||
catalog["validity"] = {r[0]: (r[2], r[3]) for r in rows}
|
||||
offset = 4 if validity else 2
|
||||
catalog["rows"] = {r[0]: {"cst": r[1], "start": (r[2] if validity else None), "end": (r[3] if validity else None), **{c: r[offset + i] for i, c in enumerate(extra)}} for r in rows}
|
||||
for value in catalog["rows"].values():
|
||||
for key in ("TIPO_ALIQUOTA", "PRED_IBS", "PRED_CBS", "IND_NFE", "IND_NFCE"):
|
||||
if key in value:
|
||||
value[key.lower()] = value.pop(key)
|
||||
for flag in ("ind_nfe", "ind_nfce"):
|
||||
if flag in value:
|
||||
value[flag] = str(value[flag]).strip() in ("1", "True", "true")
|
||||
return schema, catalog, extras
|
||||
|
||||
|
||||
def read_products(cursor, schema: dict) -> list[dict]:
|
||||
"""SELECT of only the columns that exist (old schemas keep working)."""
|
||||
have = schema.get(PRODUCT_TABLE, set())
|
||||
columns = [c for c in PRODUCT_COLUMNS if c in have]
|
||||
if "CODIGO" not in columns:
|
||||
return []
|
||||
cursor.execute(f"SELECT {', '.join(columns)} FROM {PRODUCT_TABLE} ORDER BY CODIGO")
|
||||
return [dict(zip(columns, row)) for row in cursor.fetchall()]
|
||||
|
||||
|
||||
def assess_from_connection(conn, company: dict | None, *, today: date | None = None, rtc_override: bool | None = None, ncm_valid: dict | None = None) -> dict:
|
||||
"""One read-only pass. Returns {"context": {...}, "findings": {product_code: evaluate_product(...)}}.
|
||||
rtc_override forces RTC on/off for simulations WITHOUT touching the ERP parameter (tests / what-if only).
|
||||
ncm_valid: {product_code: bool} from the decision engine (EXACT_NCM); a product whose NCM is not confirmed never gets a cClassTrib proposal."""
|
||||
cursor = conn.cursor()
|
||||
schema, catalog, extras = load_rtc_context(cursor)
|
||||
caps = detect_rtc_capabilities(schema, param_value=extras["param_value"], catalog=catalog, physical_fk=extras["physical_fk"])
|
||||
enabled = rtc_enabled(caps) if rtc_override is None else bool(rtc_override and caps["HAS_RTC_RELATIONSHIP"])
|
||||
today = today or date.today()
|
||||
cclasstrib_app = cclasstrib_applicability((company or {}).get("crt"), enabled, today)
|
||||
rtc = {"enabled": enabled, "catalog": catalog, "catalog_valid": caps["RTC_CATALOG_VALID"], "cclasstrib_applicability": cclasstrib_app}
|
||||
products = read_products(cursor, schema)
|
||||
findings = {}
|
||||
warn_by_type: Counter = Counter()
|
||||
warn_products: set = set()
|
||||
samples: list = []
|
||||
applicability: Counter = Counter()
|
||||
local_rows = (catalog or {}).get("rows") or {}
|
||||
applicable = cclasstrib_app["state"] == "REQUIRED"
|
||||
compat = compare_catalogs(local_rows, today) if (enabled and applicable and local_rows) else None
|
||||
resolver_ctx = {"rtc_enabled": enabled, "cclasstrib_applicable": applicable, "local_catalog_valid": bool(caps["RTC_CATALOG_VALID"])}
|
||||
for p in products:
|
||||
found = evaluate_product(p, company, rtc, today)
|
||||
if enabled and applicable:
|
||||
code0 = str(p.get("CODIGO") or "")
|
||||
resolution = resolve_cclasstrib({**p, "ncm_valid": bool((ncm_valid or {}).get(code0, False))}, company, resolver_ctx, local_rows, compat=compat, today=today)
|
||||
refine_rtc_finding(found, p, resolution, compat["per_code"] if compat else None)
|
||||
found["input_fingerprint"] = hashlib.sha256(json.dumps({k: (None if v is None else str(v).strip()) for k, v in p.items()} | {"rtc": enabled, "cclasstrib_applicability": cclasstrib_app}, sort_keys=True).encode()).hexdigest()
|
||||
code = str(p.get("CODIGO") or "")
|
||||
findings[code] = found
|
||||
applicability[found["applicability"]["PIS_COFINS"]] += 1
|
||||
for w in found.get("warnings", []):
|
||||
warn_by_type[w["type"]] += 1
|
||||
warn_products.add(code)
|
||||
if len(samples) < 10:
|
||||
samples.append({"product_code": code, **w})
|
||||
integrity = catalog_integrity(catalog) if catalog else None
|
||||
return {"context": {"rtc_enabled": enabled, "erp_rtc_enabled": enabled, "rtc_supported": caps["HAS_RTC_RELATIONSHIP"], "rtc_catalog_valid": caps["RTC_CATALOG_VALID"], "rtc_param": extras["param_value"],
|
||||
"rtc_source": "SIMULATION" if rtc_override is not None else "PARAMETROS.USAR_REFORMA_TRIBUTARIA", "rtc_param_rows": extras["param_rows"], "rtc_param_conflict": extras["param_conflict"],
|
||||
"rtc_observed_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "cclasstrib_applicability": cclasstrib_app,
|
||||
"legal_effective_context": legal_effective_context((company or {}).get("crt")),
|
||||
"catalog_integrity": integrity, "crt": (company or {}).get("crt"), "capabilities": caps, "products_evaluated": len(products),
|
||||
"pis_cofins_applicability": dict(applicability),
|
||||
"rtc_resolution": summarize_rtc(findings, compat) if enabled else None, "local_catalog_hash": local_catalog_hash(local_rows) if local_rows else None,
|
||||
"official_catalog": (compat or {}).get("official_catalog"),
|
||||
"non_canonical_warnings": {"products": len(warn_products), "by_type": dict(sorted(warn_by_type.items())), "samples": samples}},
|
||||
"findings": findings}
|
||||
@@ -0,0 +1,24 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_real_firebird_for_unified_fiscal(monkeypatch, request):
|
||||
"""The unified fiscal assessment opens its own READ ONLY connection; unit tests have no Firebird, so it is neutral here unless a test opts in."""
|
||||
if request.node.get_closest_marker("real_unified_fiscal"):
|
||||
return
|
||||
from gti_fiscal_agent import firebird_reader, unified_fiscal
|
||||
|
||||
class _Conn:
|
||||
def close(self):
|
||||
pass
|
||||
monkeypatch.setattr(firebird_reader, "_connect", lambda env: _Conn())
|
||||
monkeypatch.setattr(unified_fiscal, "assess_from_connection", lambda conn, company, **kw: {"context": {"rtc_enabled": False, "rtc_supported": False, "rtc_catalog_valid": False, "products_evaluated": 0}, "findings": {}})
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line("markers", "real_unified_fiscal: use the real unified fiscal assessment (with a fake cursor) instead of the neutral stub")
|
||||
@@ -0,0 +1,239 @@
|
||||
from gti_fiscal_agent import cfop_ippt_auto_apply as apply_mod
|
||||
from central.app.field_contract import ALREADY, CHANGE, MANUAL, NA
|
||||
|
||||
|
||||
class FakeFirebird:
|
||||
"""A tiny C000025: every statement is recorded; only SELECT <field> and the one allowed UPDATE per field are understood."""
|
||||
|
||||
def __init__(self, products):
|
||||
self.products = {c: dict(v) for c, v in products.items()} # code -> {"CFOP": ..., "IPPT": ...}
|
||||
self.statements, self.commits, self.rollbacks = [], 0, 0
|
||||
self._pending = {}
|
||||
|
||||
def cursor(self):
|
||||
return FakeCursor(self)
|
||||
|
||||
def commit(self):
|
||||
for (code, field), value in self._pending.items():
|
||||
self.products[code][field] = value
|
||||
self._pending = {}
|
||||
self.commits += 1
|
||||
|
||||
def rollback(self):
|
||||
self._pending = {}
|
||||
self.rollbacks += 1
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, db):
|
||||
self.db, self._rows, self.rowcount = db, [], -1
|
||||
|
||||
def fetchall(self):
|
||||
return self._rows
|
||||
|
||||
def fetchone(self):
|
||||
return self._rows[0] if self._rows else None
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
db = self.db
|
||||
db.statements.append((sql, tuple(params)))
|
||||
text = " ".join(sql.split())
|
||||
up = text.upper()
|
||||
for field in ("CFOP", "IPPT"):
|
||||
if up == f"SELECT {field} FROM C000025 WHERE CODIGO = ?":
|
||||
code = params[0]
|
||||
self._rows = [(db.products[code][field],)] if code in db.products else []
|
||||
if code in db.products and db.products[code].get("DUP"):
|
||||
self._rows = self._rows * 2
|
||||
return
|
||||
if up.startswith(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND"):
|
||||
proposed, code, expected = params
|
||||
product = db.products.get(code)
|
||||
current = (product.get(field) if product else None) or None
|
||||
hit = product is not None and current == (expected or None)
|
||||
self.rowcount = 1 if hit else 0
|
||||
if hit:
|
||||
db._pending[(code, field)] = proposed
|
||||
return
|
||||
raise AssertionError(f"unexpected SQL: {sql!r}")
|
||||
|
||||
|
||||
def candidate(code, field, expected_before, proposed, reason="X"):
|
||||
return {"product_code": code, "field": field, "expected_before": expected_before, "proposed": proposed, "reason": reason,
|
||||
"fingerprint": apply_mod.candidate_fingerprint(code, field, expected_before, proposed)}
|
||||
|
||||
|
||||
def results_for(code, field, *, state, current, proposed, write_eligible):
|
||||
return {code: {field: {"state": state, "current": current, "proposed": proposed, "write_eligible": write_eligible, "reason": "R"}}}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ write_gate (flag semantics)
|
||||
|
||||
def test_write_gate_default_off_blocks_every_mode_but_enabled_and_approval_required():
|
||||
assert apply_mod.write_gate(mode="DISABLED")["allowed"] is False
|
||||
assert apply_mod.write_gate(mode="DRY_RUN")["allowed"] is False
|
||||
assert apply_mod.write_gate(mode="APPROVAL_REQUIRED", approved=False)["allowed"] is False
|
||||
assert apply_mod.write_gate(mode="APPROVAL_REQUIRED", approved=True)["allowed"] is True
|
||||
assert apply_mod.write_gate(mode="ENABLED")["allowed"] is True
|
||||
assert apply_mod.write_gate(mode="ENABLED", capability=False)["allowed"] is False
|
||||
assert apply_mod.write_gate(mode="NOT_A_REAL_MODE")["allowed"] is False
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ select_candidates only picks AUTO_FILL_SAFE CFOP/IPPT
|
||||
|
||||
def test_select_candidates_only_change_and_write_eligible_cfop_or_ippt():
|
||||
results = {
|
||||
"000001": {"CFOP": {"state": CHANGE, "current": "5102", "proposed": "5101", "write_eligible": True, "reason": "R1"},
|
||||
"IPPT": {"state": MANUAL, "current": None, "proposed": None, "write_eligible": False, "reason": "R2"},
|
||||
"CSOSN": {"state": CHANGE, "current": None, "proposed": "102", "write_eligible": True, "reason": "R3"}}, # not CFOP/IPPT: never a candidate
|
||||
"000002": {"CFOP": {"state": ALREADY, "current": "5102", "proposed": None, "write_eligible": False, "reason": "R4"},
|
||||
"IPPT": {"state": CHANGE, "current": "T", "proposed": "P", "write_eligible": False, "reason": "R5"}}, # write_eligible=False: never a candidate
|
||||
"000003": {"IPPT": {"state": CHANGE, "current": "T", "proposed": "P", "write_eligible": True, "reason": "R6"}},
|
||||
}
|
||||
cands, total = apply_mod.select_candidates(results)
|
||||
assert total == 2
|
||||
keys = {(c["product_code"], c["field"]) for c in cands}
|
||||
assert keys == {("000001", "CFOP"), ("000003", "IPPT")}
|
||||
assert all(c["field"] in apply_mod.ALLOWED_FIELDS for c in cands)
|
||||
|
||||
|
||||
def test_weak_composed_evidence_never_becomes_a_candidate_regardless_of_the_write_flag():
|
||||
"""Missão 4 / Bloco A.3: a cell whose role evidence is weak (field_contract itself refuses to
|
||||
classify it OWN_PRODUCTION/RESALE with confidence) must never reach select_candidates as
|
||||
AUTO_FILL_SAFE - proven end-to-end from the real resolver, not just asserted in isolation.
|
||||
This holds independently of cfop_ippt_write_mode/write_gate, which only gates whether
|
||||
apply_candidates() is ever invoked - it never affects what counts as a candidate."""
|
||||
from central.app.field_contract import resolve_product, Context, WRITE_FIELDS
|
||||
|
||||
def product(**kw):
|
||||
p = {f: None for f in WRITE_FIELDS}
|
||||
p.update({"CODIGO": "000321", "PRODUTO": "CERVEJA LONG NECK NOVA SCHIN", "CODGRUPO": "G1", "TIPO": "VENDA", "PROD_COZINHA": None, "PRODUTO_SERVICO": None,
|
||||
"CLASSIFICACAO_FISCAL": "22030000", "CFOP": "5102", "CSOSN": "102", "CST": "000", "SITUACAO_TRIBUTARIA": "T", "IAT": "A", "IPPT": "T",
|
||||
"INCIDENCIA_PISCOFINS": "NENHUMA", "PISCOFINS": "N"})
|
||||
p.update(kw)
|
||||
return p
|
||||
|
||||
decision = {"status": "EXACT_NCM", "cest_status": "NO_CEST_MAPPING_FOUND", "ipi_status": "IPI_RESOLVED", "ba_st_mva_status": "NO_MVA_MAPPING_FOUND", "dataset_versions": {},
|
||||
"classification": {"ncm": "22030000", "ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.9, "rate": "3.9"}, "ipi_candidates": [], "cest": None, "cest_candidates": [], "ba_st_mva": None, "ba_st_mva_candidates": []}}
|
||||
refs = {"cfop_valid": {"5101", "5102"}, "csosn": {"102"}, "cst_icms": set(), "cst_pis": set(), "cst_cofins": set(), "origem": set("012345678"), "fcp_uf": {}, "cbs_rate_2026": None, "cclasstrib_count": 0}
|
||||
co = {"uf": "BA", "crt": "1", "contribuinte_ipi": "N", "company_code": "000001"}
|
||||
ev_weak = {"recipes": set(), "composed": set(), "composed_ratio": {"000321": {"composed": 1, "other": 19}}, "purchases": {}, "stock_entry": {"000321": True}, "stock_in": {"000321": True}}
|
||||
result = resolve_product(product(), decision, co, refs, ev_weak, Context([]))
|
||||
candidates, total = apply_mod.select_candidates({"000321": result})
|
||||
assert total == 0 and candidates == [] # the weak-evidence beer never becomes a write candidate
|
||||
|
||||
|
||||
def test_select_candidates_respects_limit():
|
||||
results = {f"{i:06d}": {"CFOP": {"state": CHANGE, "current": None, "proposed": "5102", "write_eligible": True, "reason": "R"}} for i in range(5)}
|
||||
cands, total = apply_mod.select_candidates(results, limit=2)
|
||||
assert total == 5 and len(cands) == 2
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ apply_candidate: happy path, idempotency, stale guard
|
||||
|
||||
def test_apply_candidate_writes_verifies_and_commits():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5102"}})
|
||||
c = candidate("000001", "CFOP", "5102", "5101")
|
||||
rec = apply_mod.apply_candidate(db, c, now="t0", job_id=1)
|
||||
assert rec["outcome"] == "APPLIED" and rec["old_value"] == "5102" and rec["new_value"] == "5101" and rec["verified_after"] is True
|
||||
assert db.products["000001"]["CFOP"] == "5101" and db.commits >= 1 and db.rollbacks == 0
|
||||
|
||||
|
||||
def test_second_run_over_the_same_candidate_writes_zero_no_write_required():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5101"}}) # already at the proposed value (as if a first pass already applied it)
|
||||
c = candidate("000001", "CFOP", "5102", "5101")
|
||||
try:
|
||||
apply_mod.apply_candidate(db, c, now="t0", job_id=1)
|
||||
assert False, "expected CfopIpptApplyAborted"
|
||||
except apply_mod.CfopIpptApplyAborted as e:
|
||||
assert e.reason == "NO_WRITE_REQUIRED"
|
||||
assert db.products["000001"]["CFOP"] == "5101" # unchanged, no write attempted
|
||||
|
||||
|
||||
def test_human_override_wins_stale_value_aborts_without_writing():
|
||||
db = FakeFirebird({"000001": {"CFOP": "9999"}}) # a human (or anything else) set a THIRD value since the candidate was selected
|
||||
c = candidate("000001", "CFOP", "5102", "5101") # candidate still expects the OLD "5102"
|
||||
try:
|
||||
apply_mod.apply_candidate(db, c, now="t0", job_id=1)
|
||||
assert False, "expected CfopIpptApplyAborted"
|
||||
except apply_mod.CfopIpptApplyAborted as e:
|
||||
assert e.reason == "STALE_VALUE_CHANGED_SINCE_DECISION"
|
||||
assert db.products["000001"]["CFOP"] == "9999" # the human's value is preserved, never overwritten
|
||||
|
||||
|
||||
def test_apply_candidate_blank_expected_before_and_fill():
|
||||
db = FakeFirebird({"000801": {"CFOP": None}})
|
||||
c = candidate("000801", "CFOP", None, "5102")
|
||||
rec = apply_mod.apply_candidate(db, c, now="t0", job_id=1)
|
||||
assert rec["outcome"] == "APPLIED" and rec["old_value"] is None and rec["new_value"] == "5102"
|
||||
|
||||
|
||||
def test_apply_candidate_product_not_found_or_duplicate_row_aborts():
|
||||
db = FakeFirebird({})
|
||||
try:
|
||||
apply_mod.apply_candidate(db, candidate("999999", "CFOP", None, "5102"), now="t0")
|
||||
assert False
|
||||
except apply_mod.CfopIpptApplyAborted as e:
|
||||
assert e.reason == "PRODUCT_NOT_FOUND"
|
||||
db2 = FakeFirebird({"000001": {"CFOP": "5102", "DUP": True}})
|
||||
try:
|
||||
apply_mod.apply_candidate(db2, candidate("000001", "CFOP", "5102", "5101"), now="t0")
|
||||
assert False
|
||||
except apply_mod.CfopIpptApplyAborted as e:
|
||||
assert e.reason == "PRODUCT_ROW_NOT_UNIQUE"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ apply_candidates: batch semantics, authorize() gate, flag OFF => zero writes
|
||||
|
||||
def test_apply_candidates_batch_never_stops_on_one_abort_and_reports_reasons():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5102"}, "000002": {"CFOP": "9999"}})
|
||||
cands = [candidate("000001", "CFOP", "5102", "5101"), candidate("000002", "CFOP", "5102", "5101")]
|
||||
out = apply_mod.apply_candidates(db, cands, now="t0", job_id=1)
|
||||
assert out["write_successes"] == 1 and out["write_attempts"] == 2
|
||||
assert out["aborted_reasons"] == {"STALE_VALUE_CHANGED_SINCE_DECISION": 1}
|
||||
assert db.products["000001"]["CFOP"] == "5101" and db.products["000002"]["CFOP"] == "9999"
|
||||
|
||||
|
||||
def test_flag_off_means_caller_never_invokes_apply_zero_writes():
|
||||
"""The Agent must gate BEFORE calling apply_candidates at all when write_gate(...).allowed is False -
|
||||
this is the actual "flag OFF => zero writes" contract; apply_candidates itself is only ever reached
|
||||
once the gate already said yes."""
|
||||
gate = apply_mod.write_gate(mode="DISABLED")
|
||||
assert not gate["allowed"]
|
||||
db = FakeFirebird({"000001": {"CFOP": "5102"}})
|
||||
# execute_job's real code path: `if gate["allowed"]: apply_candidates(...)` - simulate the guard directly.
|
||||
if gate["allowed"]:
|
||||
apply_mod.apply_candidates(db, [candidate("000001", "CFOP", "5102", "5101")], now="t0")
|
||||
assert db.products["000001"]["CFOP"] == "5102" and db.commits == 0
|
||||
|
||||
|
||||
def test_authorize_callback_can_veto_a_candidate_right_before_the_write():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5102"}})
|
||||
out = apply_mod.apply_candidates(db, [candidate("000001", "CFOP", "5102", "5101")], now="t0", authorize=lambda code, field: "AUTHORIZATION_CHANGED")
|
||||
assert out["write_successes"] == 0 and out["aborted_reasons"] == {"AUTHORIZATION_CHANGED": 1}
|
||||
assert db.products["000001"]["CFOP"] == "5102"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ rollback: only if the value still equals what was written
|
||||
|
||||
def test_rollback_restores_old_value_when_unchanged():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5101"}})
|
||||
item = {"product_code": "000001", "field": "CFOP", "old_value": "5102", "new_value": "5101", "audit_id": 42}
|
||||
rec = apply_mod.rollback_one(db, item, now="t1", job_id=1)
|
||||
assert rec["outcome"] == "ROLLED_BACK" and db.products["000001"]["CFOP"] == "5102"
|
||||
|
||||
|
||||
def test_rollback_refuses_when_a_human_changed_the_value_since():
|
||||
db = FakeFirebird({"000001": {"CFOP": "7777"}}) # someone changed it after the write, before the rollback
|
||||
item = {"product_code": "000001", "field": "CFOP", "old_value": "5102", "new_value": "5101", "audit_id": 42}
|
||||
rec = apply_mod.rollback_one(db, item, now="t1", job_id=1)
|
||||
assert rec["outcome"] == "ABORTED" and rec["reason"] == "ROLLBACK_CONFLICT_VALUE_CHANGED"
|
||||
assert db.products["000001"]["CFOP"] == "7777" # untouched
|
||||
|
||||
|
||||
def test_rollback_records_batch():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5101"}, "000002": {"IPPT": "P"}})
|
||||
items = [{"product_code": "000001", "field": "CFOP", "old_value": "5102", "new_value": "5101"},
|
||||
{"product_code": "000002", "field": "IPPT", "old_value": "T", "new_value": "P"}]
|
||||
out = apply_mod.rollback_records(db, items, now="t1", job_id=1)
|
||||
assert out["rolled_back"] == 2
|
||||
@@ -0,0 +1,63 @@
|
||||
import httpx
|
||||
|
||||
from gti_fiscal_agent.client import HubClient
|
||||
|
||||
|
||||
def test_identity_requests_authenticated_identity_only(monkeypatch):
|
||||
calls = {}
|
||||
payload = {
|
||||
"agent_id": 2,
|
||||
"installation_id": "00000000-0000-4000-8000-000000000222",
|
||||
"hostname": "DESKTOP-B0HL02I",
|
||||
"version": "1.1.1",
|
||||
"status": "OFFLINE",
|
||||
}
|
||||
|
||||
class Response:
|
||||
status_code = 200
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return payload
|
||||
|
||||
def fake_get(url, *, headers, timeout):
|
||||
calls.update(url=url, headers=headers, timeout=timeout)
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr(httpx, "get", fake_get)
|
||||
client = HubClient("https://hub.example", "test-agent-token")
|
||||
assert client.identity() == payload
|
||||
|
||||
assert calls["url"] == "https://hub.example/v1/agents/me/identity"
|
||||
assert calls["headers"] == {"Authorization": "Bearer test-agent-token"}
|
||||
|
||||
|
||||
def test_transport_errors_and_proxy_5xx_are_retried_but_client_errors_are_not(monkeypatch):
|
||||
import httpx
|
||||
from gti_fiscal_agent import client as c
|
||||
monkeypatch.setattr(c.time, "sleep", lambda s: None)
|
||||
calls = {"n": 0}
|
||||
|
||||
def flaky():
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise httpx.ConnectError("ssl handshake failure")
|
||||
if calls["n"] == 2:
|
||||
return httpx.Response(502)
|
||||
return httpx.Response(200, json={"ok": True})
|
||||
|
||||
assert c._resilient(flaky).status_code == 200 and calls["n"] == 3
|
||||
calls["n"] = 0
|
||||
assert c._resilient(lambda: (calls.__setitem__("n", calls["n"] + 1), httpx.Response(401))[1]).status_code == 401 and calls["n"] == 1 # 4xx: no retry
|
||||
calls["n"] = 0
|
||||
|
||||
def always_down():
|
||||
calls["n"] += 1
|
||||
raise httpx.ConnectError("down")
|
||||
|
||||
import pytest
|
||||
with pytest.raises(httpx.ConnectError):
|
||||
c._resilient(always_down)
|
||||
assert calls["n"] == 10 # bounded: 10 attempts (~3 min), then the job fails cleanly
|
||||
@@ -0,0 +1,134 @@
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent.company_context import CompanyContextError, config_from_env, mask_cnpj, preflight_report, resolve
|
||||
from gti_fiscal_agent.firebird_reader import parse_company_context
|
||||
|
||||
CN_A, CN_B, CN_C = "11111111000111", "22222222000122", "33333333000133"
|
||||
|
||||
|
||||
def co(code, cnpj, uf="BA", crt="1", optante="S"):
|
||||
return {"code": code, "cnpj": cnpj, "uf": uf, "crt": crt, "optante": optante, "name": "X"}
|
||||
|
||||
|
||||
def ev(companies, emissor=None, docs=None):
|
||||
return {"c000004": companies, "emissor": emissor if emissor is not None else [], "nfce_keys": docs or [], "nf_by_filial": {}}
|
||||
|
||||
|
||||
def emis(cnpj=CN_A, uf="BA", crt="1"):
|
||||
return {"cnpj": cnpj, "uf": uf, "crt": crt}
|
||||
|
||||
|
||||
DOC_A = [{"cuf": "29", "cnpj": CN_A, "n": 100}]
|
||||
|
||||
|
||||
def test_single_company_all_sources_agree_is_auto_resolved():
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], DOC_A))
|
||||
assert r["status"] == "PASS" and r["classification"] == "AUTO_RESOLVED" and r["company_uf"] == "BA" and r["company_crt"] == "1" and r["confidence"] == "HIGH"
|
||||
assert "DOCUMENT_KEYS" in r["source_of_truth"]["uf"]
|
||||
|
||||
|
||||
def test_multiple_rows_resolved_only_through_emitter_identity_never_first_row():
|
||||
rows = [co("000001", CN_B, "SP", None, "N"), co("000005", CN_A), co("000007", CN_C, "BA", None, "N")]
|
||||
r = resolve(ev(rows, [emis(CN_A)], DOC_A))
|
||||
assert r["status"] == "PASS" and r["company_code"] == "000005" and r["source_of_truth"]["company"] == "EMISSOR_CNPJ_MATCH"
|
||||
|
||||
|
||||
def test_multiple_rows_without_disambiguation_requires_config():
|
||||
rows = [co("000001", CN_A), co("000002", CN_B)]
|
||||
r = resolve(ev(rows, []))
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFIG_REQUIRED" and r["company_code"] is None
|
||||
assert resolve(ev(rows, [emis(CN_C)]))["classification"] == "CONFIG_REQUIRED" # emitter matches none
|
||||
ok = resolve(ev(rows, [], []), {"company_code": "000002"})
|
||||
assert ok["status"] == "PASS" and ok["company_code"] == "000002" and ok["classification"] == "CONFIGURED"
|
||||
|
||||
|
||||
def test_configured_company_must_exist_and_agree_with_emitter():
|
||||
rows = [co("000001", CN_A), co("000002", CN_B)]
|
||||
assert resolve(ev(rows, [emis(CN_A)]), {"company_cnpj": "99999999999999"})["classification"] == "CONFIG_REQUIRED"
|
||||
r = resolve(ev(rows, [emis(CN_A)], DOC_A), {"company_code": "000002"})
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" # selected company differs from the emitter: unexplained
|
||||
|
||||
|
||||
def test_documents_only_from_another_cnpj_conflict_but_mixed_history_is_a_warning():
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], [{"cuf": "29", "cnpj": CN_B, "n": 10}]))
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT"
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], [{"cuf": "29", "cnpj": CN_A, "n": 10}, {"cuf": "29", "cnpj": CN_B, "n": 1}]))
|
||||
assert r["status"] == "PASS" and any("HISTORICAL_NFCE_KEYS_FROM_OTHER_CNPJ" in w for w in r["warnings"])
|
||||
|
||||
|
||||
def test_uf_conflict_and_unknown():
|
||||
assert resolve(ev([co("1", CN_A, "BA")], [emis(uf="SP")]))["classification"] == "CONFLICT"
|
||||
assert resolve(ev([co("1", CN_A, "BA")], [emis()], [{"cuf": "35", "cnpj": CN_A, "n": 5}]))["classification"] == "CONFLICT" # key says SP
|
||||
assert resolve(ev([co("1", CN_A, "")], [emis(uf="")]))["classification"] == "UNRESOLVED"
|
||||
|
||||
|
||||
def test_crt_conflict_needs_explicit_source_and_records_it():
|
||||
base = ev([co("1", CN_A, crt="3")], [emis(crt="1")], DOC_A)
|
||||
r = resolve(base)
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" and r["crt_candidates"] == {"EMISSOR": "1", "C000004": "3"}
|
||||
ok = resolve(base, {"crt_source": "EMISSOR"})
|
||||
assert ok["status"] == "PASS" and ok["company_crt"] == "1" and ok["classification"] == "CONFIGURED" and ok["confidence"] == "MEDIUM"
|
||||
assert resolve(base, {"crt_source": "C000004"})["company_crt"] == "3"
|
||||
assert resolve(base, {"crt_source_invalid": True})["classification"] == "CONFIG_REQUIRED"
|
||||
|
||||
|
||||
def test_crt_unknown_fails_and_single_source_is_flagged():
|
||||
assert resolve(ev([co("1", CN_A, crt=None)], [emis(crt="")]))["classification"] == "UNRESOLVED"
|
||||
r = resolve(ev([co("1", CN_A, crt=None)], [emis(crt="1")]))
|
||||
assert r["status"] == "PASS" and "CRT_HAS_A_SINGLE_SOURCE" in r["warnings"]
|
||||
|
||||
|
||||
def test_optante_inconsistency_is_a_warning_not_a_guess():
|
||||
r = resolve(ev([co("1", CN_A, crt="3", optante="S")], [emis(crt="3")]))
|
||||
assert r["status"] == "PASS" and any("OPTANTE_SIMPLES" in w for w in r["warnings"])
|
||||
|
||||
|
||||
def test_contribuinte_ipi_is_auto_discovered_from_the_same_selected_c000004_row():
|
||||
"""C000004.CONTRIBUINTE_IPI (real column, same row already proven to be THE company by CRT/UF resolution) closes the fiscal-completion gap
|
||||
where CST_IPI could not be resolved without asking an operator for a fact the ERP already proves."""
|
||||
row = co("000001", CN_A)
|
||||
row["contribuinte_ipi"] = "N"
|
||||
r = resolve(ev([row], [emis()], DOC_A))
|
||||
assert r["company_contribuinte_ipi"] == "N" and r["source_of_truth"]["contribuinte_ipi"] == "C000004"
|
||||
|
||||
|
||||
def test_contribuinte_ipi_absent_or_invalid_stays_unresolved_never_guessed():
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], DOC_A)) # co() never sets contribuinte_ipi
|
||||
assert r["company_contribuinte_ipi"] is None
|
||||
row = co("000002", CN_B); row["contribuinte_ipi"] = "X" # garbage value: never trusted
|
||||
r2 = resolve(ev([row], [emis(cnpj=CN_B)], [{"cuf": "29", "cnpj": CN_B, "n": 1}]))
|
||||
assert r2["company_contribuinte_ipi"] is None
|
||||
|
||||
|
||||
def test_full_cnpj_is_never_exposed():
|
||||
r = resolve(ev([co("1", CN_A)], [emis()], DOC_A))
|
||||
dump = str(preflight_report(r)) + str(r)
|
||||
assert CN_A not in dump and mask_cnpj(CN_A) == "11**********11"
|
||||
|
||||
|
||||
def test_read_company_context_fails_closed(monkeypatch):
|
||||
from gti_fiscal_agent import company_context as cc, firebird_reader as fr
|
||||
monkeypatch.setattr(cc, "preflight_company_context", lambda env: resolve(ev([co("1", CN_A), co("2", CN_B)], [])))
|
||||
with pytest.raises(CompanyContextError):
|
||||
fr.read_company_context({})
|
||||
monkeypatch.setattr(cc, "preflight_company_context", lambda env: resolve(ev([co("1", CN_A)], [emis()], DOC_A)))
|
||||
assert fr.read_company_context({})["uf"] == "BA"
|
||||
|
||||
|
||||
def test_config_from_env_and_legacy_parser_unchanged():
|
||||
assert config_from_env({"COMPANY_CODE": " 000005 ", "COMPANY_CNPJ": "11.111.111/0001-11", "COMPANY_CRT_SOURCE": "emissor"}) == {"company_code": "000005", "company_cnpj": "11111111000111", "crt_source": "EMISSOR", "crt_source_invalid": False, "uf_override": None}
|
||||
assert config_from_env({"COMPANY_CRT_SOURCE": "x"})["crt_source_invalid"] is True
|
||||
assert parse_company_context([("ba", "1")]) == {"uf": "BA", "crt": "1"}
|
||||
|
||||
|
||||
def test_auto_crt_source_and_uf_override_semantics():
|
||||
assert config_from_env({"COMPANY_CRT_SOURCE": "AUTO"}) ["crt_source"] is None and config_from_env({"COMPANY_CRT_SOURCE": "AUTO"})["crt_source_invalid"] is False
|
||||
conflict = ev([co("1", CN_A, crt="3")], [emis(crt="1")], DOC_A)
|
||||
assert resolve(conflict, config_from_env({"COMPANY_CRT_SOURCE": "AUTO"}))["status"] == "FAIL" # AUTO never hides a CRT conflict
|
||||
good = ev([co("1", CN_A)], [emis()], DOC_A)
|
||||
assert resolve(good, {"uf_override": "BA"})["status"] == "PASS"
|
||||
r = resolve(good, {"uf_override": "SP"})
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" # override never replaces the resolver silently
|
||||
blank = ev([co("1", CN_A, uf="")], [emis(uf="")])
|
||||
r = resolve(blank, {"uf_override": "BA"})
|
||||
assert r["status"] == "PASS" and "UF_FROM_EXPLICIT_OVERRIDE" in r["warnings"]
|
||||
@@ -0,0 +1,179 @@
|
||||
"""DPAPI LocalMachine credential store: encrypt/decrypt round-trip, plaintext
|
||||
fallback is dev-only, ACL is SYSTEM/Administrators, and the token is never
|
||||
written into agent.env."""
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent import credential_store
|
||||
|
||||
import win32api
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
RUN_SCRIPT = AGENT_ROOT / "scripts" / "run_agent_service.py"
|
||||
|
||||
|
||||
def load_run_service():
|
||||
spec = importlib.util.spec_from_file_location("run_agent_service_cred", RUN_SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def write_plaintext(path: Path, value: str = "TOKEN_PLAINTEXT_DEV"):
|
||||
path.write_text(value, encoding="utf-8")
|
||||
|
||||
|
||||
# ---------- DPAPI round-trip ----------
|
||||
|
||||
def _allow_current_user(path: Path) -> None:
|
||||
"""The strict SYSTEM/Administrators DACL blocks the pytest user (not an
|
||||
admin here). As the file OWNER the current user may grant itself read —
|
||||
exactly like the elevated installer/local administrator does on the real
|
||||
host before the credential is consumed by the SYSTEM service."""
|
||||
import subprocess
|
||||
|
||||
import win32security
|
||||
|
||||
acct_sid, _domain, _type = win32security.LookupAccountName(None, win32api.GetUserName())
|
||||
sid_str = win32security.ConvertSidToStringSid(acct_sid)
|
||||
subprocess.run(["icacls", str(path), "/grant", f"*{sid_str}:(R)"], check=True, capture_output=True)
|
||||
|
||||
|
||||
def test_write_then_read_roundtrip(tmp_path):
|
||||
path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(path, "TOKEN_DPAPI_1")
|
||||
_allow_current_user(path)
|
||||
assert credential_store.read_credential(path) == "TOKEN_DPAPI_1"
|
||||
|
||||
|
||||
def test_stored_blob_never_contains_the_plaintext_secret(tmp_path):
|
||||
path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(path, "TOKEN_VERY_SECRET_9")
|
||||
_allow_current_user(path)
|
||||
raw = path.read_bytes()
|
||||
assert b"TOKEN_VERY_SECRET_9" not in raw
|
||||
|
||||
|
||||
def test_read_missing_file_raises(tmp_path):
|
||||
with pytest.raises(credential_store.CredentialStoreError):
|
||||
credential_store.read_credential(tmp_path / "nope.bin")
|
||||
|
||||
|
||||
def test_write_detects_dpapi_unavailable_and_fails_closed(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(credential_store, "_dpapi_available", lambda: False)
|
||||
with pytest.raises(credential_store.CredentialStoreError):
|
||||
credential_store.write_credential(tmp_path / "credentials.bin", "never")
|
||||
|
||||
|
||||
# ---------- plaintext fallback (dev-only) ----------
|
||||
|
||||
def test_plaintext_rejected_by_default(tmp_path):
|
||||
path = tmp_path / "credentials.bin"
|
||||
write_plaintext(path)
|
||||
with pytest.raises(credential_store.CredentialStoreError):
|
||||
credential_store.read_credential(path)
|
||||
|
||||
|
||||
def test_plaintext_accepted_only_with_dev_flag(tmp_path, monkeypatch):
|
||||
path = tmp_path / "credentials.bin"
|
||||
write_plaintext(path, "TOKEN_DEV_PLAIN")
|
||||
assert credential_store.read_credential(path, env={"GTI_ALLOW_PLAINTEXT_CREDENTIAL": "1"}) == "TOKEN_DEV_PLAIN"
|
||||
monkeypatch.setenv("GTI_ALLOW_PLAINTEXT_CREDENTIAL", "1")
|
||||
assert credential_store.read_credential(path) == "TOKEN_DEV_PLAIN"
|
||||
monkeypatch.delenv("GTI_ALLOW_PLAINTEXT_CREDENTIAL", raising=False)
|
||||
with pytest.raises(credential_store.CredentialStoreError):
|
||||
credential_store.read_credential(path)
|
||||
|
||||
|
||||
# ---------- ACL (SYSTEM/Administrators only) ----------
|
||||
|
||||
def test_write_sets_system_and_administrators_acl(tmp_path):
|
||||
import win32security
|
||||
|
||||
path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(path, "TOKEN_ACL_7")
|
||||
admin_sid = win32security.ConvertStringSidToSid("S-1-5-32-544")
|
||||
system_sid = win32security.ConvertStringSidToSid("S-1-5-18")
|
||||
sd = win32security.GetFileSecurity(str(path), win32security.DACL_SECURITY_INFORMATION)
|
||||
dacl = sd.GetSecurityDescriptorDacl()
|
||||
assert dacl.GetAceCount() == 2
|
||||
allow_trustees = []
|
||||
for i in range(dacl.GetAceCount()):
|
||||
ace = dacl.GetAce(i)
|
||||
assert ace[0][0] == win32security.ACCESS_ALLOWED_ACE_TYPE, ace # only allows, no deny
|
||||
allow_trustees.append(win32security.ConvertSidToStringSid(ace[2]))
|
||||
assert set(allow_trustees) == {"S-1-5-18", "S-1-5-32-544"}
|
||||
|
||||
|
||||
# ---------- install_env_from_credentials / Config integration ----------
|
||||
|
||||
def test_install_env_from_credentials_fills_agent_token(tmp_path):
|
||||
path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(path, "TOKEN_FROM_FILE")
|
||||
_allow_current_user(path)
|
||||
env = {"AGENT_TOKEN": "", "HUB_URL": "https://x"}
|
||||
install_env_from_credentials = credential_store.install_env_from_credentials
|
||||
assert install_env_from_credentials(path, env) == "TOKEN_FROM_FILE"
|
||||
assert env["AGENT_TOKEN"] == "TOKEN_FROM_FILE"
|
||||
|
||||
|
||||
def test_install_env_prefers_explicit_agent_token(tmp_path):
|
||||
path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(path, "TOKEN_FROM_FILE")
|
||||
env = {"AGENT_TOKEN": "TOKEN_FROM_ENV"}
|
||||
install_env_from_credentials = credential_store.install_env_from_credentials
|
||||
assert install_env_from_credentials(path, env) == "TOKEN_FROM_ENV"
|
||||
assert env["AGENT_TOKEN"] == "TOKEN_FROM_ENV"
|
||||
|
||||
|
||||
def test_config_reads_token_from_credential_file(tmp_path, monkeypatch):
|
||||
credential_path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(credential_path, "TOKEN_CRED_FILE")
|
||||
_allow_current_user(credential_path)
|
||||
env = {
|
||||
"FIREBIRD_DATABASE_PATH": str(tmp_path / "nope.fdb"),
|
||||
"FIREBIRD_WRITES": "0",
|
||||
"FISCAL_MODE": "DRY_RUN",
|
||||
"GTI_CREDENTIAL_FILE": str(credential_path),
|
||||
}
|
||||
for key in list(env):
|
||||
monkeypatch.setenv(key, env[key])
|
||||
service = load_run_service()
|
||||
config = service.Config(os.environ)
|
||||
assert config.agent_token == "TOKEN_CRED_FILE"
|
||||
assert config.firebird_writes is False # FIREBIRD_WRITES stays 0 regardless of credential
|
||||
|
||||
|
||||
# ---------- no-write enforcement (drift) ----------
|
||||
|
||||
def test_firebird_writes_fail_closed_with_credential_file(tmp_path, monkeypatch):
|
||||
credential_path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(credential_path, "TOKEN")
|
||||
_allow_current_user(credential_path)
|
||||
env = {
|
||||
"FIREBIRD_DATABASE_PATH": str(tmp_path / "x.fdb"),
|
||||
"FIREBIRD_WRITES": "0",
|
||||
"GTI_CREDENTIAL_FILE": str(credential_path),
|
||||
}
|
||||
for key in list(env):
|
||||
monkeypatch.setenv(key, env[key])
|
||||
service = load_run_service()
|
||||
|
||||
# Even a hostile FISCAL_MODE cannot enable writes when FIREBIRD_WRITES=0.
|
||||
monkeypatch.setenv("FISCAL_MODE", "AUTO_APPLY")
|
||||
config = service.Config(os.environ)
|
||||
assert config.firebird_writes is False
|
||||
|
||||
|
||||
def test_credential_file_default_lives_under_program_files():
|
||||
default = credential_store.default_credential_path()
|
||||
assert "Program Files" in str(default)
|
||||
assert default.name == "credentials.bin"
|
||||
@@ -0,0 +1,407 @@
|
||||
"""Agent enrichment: baseline/new-product detection, feature gating, typed write, second run = 0 writes, human override (incl. deleted value)."""
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent import enrichment as E # noqa: E402
|
||||
|
||||
|
||||
class Conn:
|
||||
"""sqlite stand-in for the Firebird connection; close() is a no-op so the worker can 'open' it repeatedly."""
|
||||
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
def cursor(self):
|
||||
return self.db.cursor()
|
||||
|
||||
def commit(self):
|
||||
self.db.commit()
|
||||
|
||||
def rollback(self):
|
||||
self.db.rollback()
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class Hub:
|
||||
def __init__(self, features, decisions=None, state="EXACT_GTIN_VERIFIED"):
|
||||
self.features = features
|
||||
self.decisions = decisions or {}
|
||||
self.state = state
|
||||
self.resolve_calls, self.reports, self.overridden = [], [], {}
|
||||
|
||||
def authorizations(self):
|
||||
return {"features": dict(self.features)}
|
||||
|
||||
def resolve_product(self, payload):
|
||||
self.resolve_calls.append(payload)
|
||||
fields = {k: dict(v) for k, v in self.decisions.items()}
|
||||
for f in list(fields):
|
||||
if f in self.overridden.get(payload["product_code"], set()):
|
||||
fields[f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
|
||||
elif payload["current"].get(f) and not E.is_placeholder(f, payload["current"].get(f)):
|
||||
fields[f] = {"status": "ALREADY_SET", "value": None}
|
||||
return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}}
|
||||
|
||||
def pending_enrichment(self):
|
||||
codes, self.flagged = list(getattr(self, "flagged", [])), []
|
||||
return codes
|
||||
|
||||
def report_enrichment(self, payload):
|
||||
self.reports.append(payload)
|
||||
for item in payload["fields"]:
|
||||
if item["outcome"] == "HUMAN_OVERRIDDEN":
|
||||
self.overridden.setdefault(payload["product_code"], set()).add(item["field"])
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
ENABLED = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
|
||||
DECISIONS = {"CLASSIFICACAO_FISCAL": {"status": "AUTO_FILL_SAFE", "value": "22021000", "rule": "NCM_KB_VERIFIED", "kb_status": "VERIFIED_KB"},
|
||||
"CEST": {"status": "AUTO_FILL_SAFE", "value": "0300700", "rule": "CEST_OFFICIAL_UNIQUE_MAPPING"},
|
||||
"IPI": {"status": "AUTO_FILL_SAFE", "value": "0.0", "rule": "IPI_TIPI_NUMERIC"}}
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def env(tmp_path):
|
||||
db = sqlite3.connect(":memory:", check_same_thread=False)
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT, CODBARRA TEXT, "
|
||||
+ ", ".join(f"{f} TEXT" for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS) + ")")
|
||||
db.executemany("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", [(f"H{i}", f"HIST {i}", "", "keep") for i in range(50)])
|
||||
db.commit()
|
||||
hub = Hub(dict(ENABLED), DECISIONS)
|
||||
store = E.EnrichmentStore(tmp_path / "e.db")
|
||||
now = [1000.0]
|
||||
reproc = []
|
||||
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=reproc.append, clock=lambda: now[0], sleep=lambda s: None)
|
||||
return db, hub, store, worker, now, reproc
|
||||
|
||||
|
||||
def add(db, code, name="REFRIGERANTE COLA LATA 350ML", gtin="7891000100103"):
|
||||
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", (code, name, gtin, "keep"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def row(db, code):
|
||||
return db.execute("SELECT CLASSIFICACAO_FISCAL,CEST,IPI,OTHER FROM C000025 WHERE CODIGO=?", (code,)).fetchone()
|
||||
|
||||
|
||||
def test_baseline_never_sends_historical_products(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
assert worker.step() == [] # baseline
|
||||
now[0] += 5
|
||||
assert worker.step() == [] and hub.resolve_calls == []
|
||||
assert len(store.known()) == 50
|
||||
|
||||
|
||||
def test_new_product_is_enriched_only_in_empty_fields_then_second_run_zero_writes(env):
|
||||
db, hub, store, worker, now, reproc = env
|
||||
worker.step()
|
||||
add(db, "N1")
|
||||
now[0] += 5
|
||||
assert worker.step() == ["N1"]
|
||||
assert row(db, "N1") == ("22021000", "0300700", 0.0, "keep")
|
||||
assert reproc == [["N1"]]
|
||||
assert row(db, "H1") == (None, None, None, "keep") # historical untouched
|
||||
assert [r["outcome"] for r in hub.reports[0]["fields"]] == ["APPLIED"] * 3
|
||||
now[0] += 5
|
||||
before = db.total_changes
|
||||
worker.step()
|
||||
assert db.total_changes == before and len(hub.resolve_calls) == 1 # nothing re-processed, 0 writes
|
||||
|
||||
|
||||
def test_current_report_includes_fiscal_completion_and_classification_columns(tmp_path):
|
||||
"""`current` sent to the Hub used to carry only CLASSIFICACAO_FISCAL/CEST/IPI. field_contract.resolve_product (now wired into
|
||||
/v1/products/resolve) also needs the current CFOP/CST/CSOSN/... values and the classification columns infer_role reads (TIPO/
|
||||
PROD_COZINHA/PRODUTO_SERVICO/CODGRUPO) - otherwise it can never see what's already filled or auto-discover product_role."""
|
||||
db = sqlite3.connect(":memory:", check_same_thread=False)
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT, CODBARRA TEXT, "
|
||||
+ ", ".join(f"{f} TEXT" for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS) + ")")
|
||||
db.commit()
|
||||
hub = Hub(dict(ENABLED), DECISIONS)
|
||||
store = E.EnrichmentStore(tmp_path / "e.db")
|
||||
now = [1000.0]
|
||||
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=lambda c: None, clock=lambda: now[0], sleep=lambda s: None)
|
||||
worker.step() # baseline
|
||||
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER,CSOSN,TIPO,PRODUTO_SERVICO) VALUES(?,?,?,?,?,?,?)", ("N9", "PRODUTO X", "7891000100103", "keep", "102", "VENDA", None))
|
||||
db.commit()
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
sent = hub.resolve_calls[0]["current"]
|
||||
assert sent["CSOSN"] == "102" and sent["TIPO"] == "VENDA"
|
||||
for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS:
|
||||
assert f in sent, f
|
||||
assert hub.resolve_calls[0]["ev"] == {"recipes": [], "composed": [], "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}} # targeted evidence rides along (empty here: sqlite has no RDB$RELATION_FIELDS)
|
||||
|
||||
|
||||
def test_existing_value_is_never_overwritten(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N2")
|
||||
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N2'")
|
||||
db.commit()
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "N2")[0] == "19059090" and row(db, "N2")[1] == "0300700"
|
||||
|
||||
|
||||
def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
for features in ({"features_configured": False}, {"features_configured": True, "enrichment_mode": "DISABLED"}):
|
||||
hub.features = features
|
||||
worker._features_at = -999
|
||||
assert worker.step() == [] and hub.resolve_calls == []
|
||||
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
|
||||
worker._features_at = -999
|
||||
now[0] += 5
|
||||
worker.step() # re-baseline after the inactive period
|
||||
add(db, "N3")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "N3") == (None, None, None, "keep") and hub.reports[0]["fields"][0]["outcome"] == "DRY_RUN"
|
||||
|
||||
|
||||
def test_authorization_revoked_right_before_write_fails_closed(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N4")
|
||||
calls = {"n": 0}
|
||||
|
||||
def flip():
|
||||
calls["n"] += 1
|
||||
return {"features": dict(ENABLED if calls["n"] == 1 else {"features_configured": True, "enrichment_mode": "DISABLED"})}
|
||||
hub.authorizations = flip
|
||||
now[0] += 5
|
||||
worker._features_at = -999
|
||||
worker.step()
|
||||
assert row(db, "N4") == (None, None, None, "keep")
|
||||
assert all(f["outcome"] == "ABORTED" and f["reason"] == "AUTHORIZATION_CHANGED" for f in hub.reports[0]["fields"])
|
||||
|
||||
|
||||
def test_human_override_including_deleted_value_is_final(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N5")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
db.execute("UPDATE C000025 SET CEST='', CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N5'") # a person deletes the CEST and changes the NCM
|
||||
db.commit()
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert store.is_overridden("N5", "CEST") and store.is_overridden("N5", "CLASSIFICACAO_FISCAL") and not store.is_overridden("N5", "IPI")
|
||||
# the description changes a moment later: the worker re-resolves, but overridden fields must NOT be written again
|
||||
db.execute("UPDATE C000025 SET PRODUTO='REFRIGERANTE COLA LATA 350ML ZERO' WHERE CODIGO='N5'")
|
||||
db.commit()
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert row(db, "N5")[:2] == ("19059090", "")
|
||||
|
||||
|
||||
def test_write_field_allowlist_and_shape():
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT, COD_ORIGEM_MERCADORIA TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
||||
with pytest.raises(ValueError):
|
||||
E.write_field(Conn(db), "1", "COD_ORIGEM_MERCADORIA", "1")
|
||||
with pytest.raises(ValueError):
|
||||
E.write_field(Conn(db), "1", "CCLASSTRIB", "000001")
|
||||
db.execute("INSERT INTO C000025(CODIGO) VALUES('1')")
|
||||
db.commit()
|
||||
assert E.write_field(Conn(db), "1", "CLASSIFICACAO_FISCAL", "123")["reason"] == "VALUE_SHAPE_INVALID"
|
||||
|
||||
|
||||
def test_hub_offline_keeps_product_pending_and_retries(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N6")
|
||||
|
||||
def boom(payload):
|
||||
raise ConnectionError("down")
|
||||
good = hub.resolve_product
|
||||
hub.resolve_product = boom
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert store.get("N6")["state"] == "RETRY" and row(db, "N6")[0] is None
|
||||
hub.resolve_product = good
|
||||
now[0] += 40
|
||||
assert worker.step() == ["N6"] and row(db, "N6")[0] == "22021000"
|
||||
|
||||
|
||||
def test_gtin_check_parity_with_the_hub_and_codbarra_is_used():
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from central.app.enrichment.gtin import normalize_gtin
|
||||
for value in ("7891000100103", "7891000100104", "", "0", "12345678", "40170725", "036000291452", "11111111", "78910001001030", "abc", "2000000000008"):
|
||||
assert E.gtin_check_ok(value) == (normalize_gtin(value)["status"] in ("VALID_GTIN", "RESTRICTED_GTIN")), value
|
||||
assert E.pick_gtin({"GTIN": "", "CODBARRA": "7891000100103"}) == "7891000100103"
|
||||
assert E.pick_gtin({"GTIN": "7891000100103", "CODBARRA": "999"}) == "7891000100103"
|
||||
assert E.pick_gtin({"GTIN": "", "CODBARRA": "123"}) == "123" and E.pick_gtin({}) == ""
|
||||
|
||||
|
||||
def test_products_created_while_inactive_are_not_new_when_reactivated(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step() # baseline
|
||||
hub.features = {"features_configured": True, "enrichment_mode": "DISABLED"}
|
||||
worker._features_at = -999
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
add(db, "W1") # created while the function is off
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
hub.features = dict(ENABLED)
|
||||
worker._features_at = -999
|
||||
now[0] += 5
|
||||
assert worker.step() == [] and hub.resolve_calls == [] # re-baselined: W1 is history now
|
||||
add(db, "W2")
|
||||
now[0] += 5
|
||||
assert worker.step() == ["W2"]
|
||||
|
||||
|
||||
# ---- TECHNICAL_PLACEHOLDER (the SGV refuses a product without NCM: operators type 00000000) ---------------------------------------------------
|
||||
def add_with_ncm(db, code, ncm, name="COCA-COLA ORIGINAL LATA 350 ML"):
|
||||
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,CLASSIFICACAO_FISCAL,OTHER) VALUES(?,?,?,?,?)", (code, name, "7891000100103", ncm, "keep"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_placeholder_parity_with_the_hub():
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from central.app.enrichment import placeholders as hub
|
||||
assert {k: set(v) for k, v in E.PLACEHOLDERS.items()} == {k: set(v) for k, v in hub.PRODUCT_ENRICHMENT_PLACEHOLDERS.items()}
|
||||
for value in ("00000000", "0000.00.00", " 00000000 ", "22021000", "99999999", "0", "SEM NCM", "", None):
|
||||
assert E.is_placeholder("CLASSIFICACAO_FISCAL", value) == hub.is_placeholder("CLASSIFICACAO_FISCAL", value), value
|
||||
assert not E.is_placeholder("CEST", "0000000") and not E.is_placeholder("IPI", 0)
|
||||
|
||||
|
||||
def test_placeholder_ncm_is_replaced_dry_run_first_then_enabled_and_second_run_is_zero(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
|
||||
worker._features_at = -999
|
||||
worker.step() # baseline
|
||||
add_with_ncm(db, "P1", "00000000")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "P1")[0] == "00000000" and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN" # DRY_RUN: detected, zero writes
|
||||
assert hub.resolve_calls[-1]["current"]["CLASSIFICACAO_FISCAL"] == "00000000"
|
||||
hub.features = dict(ENABLED)
|
||||
worker._features_at = -999
|
||||
worker._auth_cache = (None, 0.0)
|
||||
add_with_ncm(db, "P2", "00000000")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "P2") == ("22021000", "0300700", 0.0, "keep")
|
||||
first = [f for f in hub.reports[-1]["fields"] if f["field"] == "CLASSIFICACAO_FISCAL"][0]
|
||||
assert first["outcome"] == "APPLIED" and first["old"] == "00000000"
|
||||
before = db.total_changes
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
now[0] += 60
|
||||
worker.step()
|
||||
assert db.total_changes == before # second run: 0 writes
|
||||
|
||||
|
||||
def test_real_or_unlisted_ncm_is_never_replaced(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
|
||||
add_with_ncm(db, code, ncm)
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
|
||||
assert row(db, code)[0] == ncm, code # EXISTING_REAL_VALUE_PRESERVED, even if the KB says otherwise
|
||||
|
||||
|
||||
def test_write_field_guards_the_placeholder_swap():
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
||||
db.executemany("INSERT INTO C000025 VALUES(?,?)", [("a", "00000000"), ("b", "22021000"), ("c", None), ("d", "0000.00.00")])
|
||||
db.commit()
|
||||
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "19059090")["reason"] == "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"
|
||||
assert E.write_field(Conn(db), "c", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert E.write_field(Conn(db), "d", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert [r[0] for r in db.execute("SELECT CLASSIFICACAO_FISCAL FROM C000025 ORDER BY CODIGO")] == ["19059090", "22021000", "19059090", "19059090"]
|
||||
|
||||
|
||||
# ---- restart / race / multiple products / stale guard / reevaluation / provenance --------------------------------------------------------------
|
||||
def make_worker(db, hub, store, now):
|
||||
return E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=lambda codes: None, clock=lambda: now[0], sleep=lambda s: None)
|
||||
|
||||
|
||||
def test_product_created_while_agent_is_down_is_found_after_restart(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "D1") # created while the Agent is stopped / restarting
|
||||
worker2 = make_worker(db, hub, store, now) # new process, same persisted state
|
||||
now[0] += 5
|
||||
assert worker2.step() == ["D1"] and row(db, "D1")[0] == "22021000"
|
||||
|
||||
|
||||
def test_many_products_between_polls_and_a_deleted_one(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
for i in range(5):
|
||||
add(db, f"M{i}")
|
||||
db.execute("DELETE FROM C000025 WHERE CODIGO='M2'")
|
||||
db.commit()
|
||||
add(db, "M9")
|
||||
now[0] += 5
|
||||
done = worker.step()
|
||||
assert sorted(done) == ["M0", "M1", "M3", "M4", "M9"] and all(row(db, c)[0] == "22021000" for c in done)
|
||||
|
||||
|
||||
def test_reevaluation_flagged_by_the_hub_retries_an_unresolved_product(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
hub.decisions = {} # first attempt: nothing known yet
|
||||
add(db, "U1")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "U1")[0] is None and store.get("U1")["state"] == "DONE"
|
||||
hub.decisions = dict(DECISIONS) # new evidence (KB decision) + 'Reavaliar'/'Reprocessar'
|
||||
hub.flagged = ["U1"]
|
||||
now[0] += 30
|
||||
assert "U1" in worker.step() and row(db, "U1")[0] == "22021000"
|
||||
now[0] += 30
|
||||
before = db.total_changes
|
||||
worker.step()
|
||||
assert db.total_changes == before
|
||||
|
||||
|
||||
def test_stale_write_is_refused_when_a_person_changed_the_value_after_the_decision(env):
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
||||
db.execute("INSERT INTO C000025 VALUES('a','19059090')")
|
||||
db.execute("INSERT INTO C000025 VALUES('b','00000000')")
|
||||
db.commit()
|
||||
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "22021000", observed="00000000", guard_observed=True)["reason"] == "STALE_CONFLICT"
|
||||
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "22021000", observed="00000000", guard_observed=True)["outcome"] == "APPLIED"
|
||||
|
||||
|
||||
def test_placeholder_after_a_human_override_is_not_filled_again(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add_with_ncm(db, "HP1", "00000000") # born with the placeholder: eligible
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "HP1")[0] == "22021000"
|
||||
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='00000000' WHERE CODIGO='HP1'") # a person puts the SAME value back: a human decision now
|
||||
db.commit()
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert store.is_overridden("HP1", "CLASSIFICACAO_FISCAL")
|
||||
hub.flagged = ["HP1"]
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert row(db, "HP1")[0] == "00000000" # provenance matters: same value, different meaning
|
||||
|
||||
|
||||
def test_automatic_mode_needs_no_sync_button_or_restart(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "A1")
|
||||
now[0] += 3
|
||||
assert worker.step() == ["A1"] and row(db, "A1")[0] == "22021000" # save -> next poll -> written
|
||||
@@ -0,0 +1,85 @@
|
||||
"""read_product_evidence: targeted, single-product evidence for the ad-hoc /v1/products/resolve call - the same recipes/composed/purchases/
|
||||
stock facts read_evidence already gathers for a full sync job, scoped to ONE product_code, never a full-table scan. Measured against a real
|
||||
production-equivalent database (D:\\PROJETOS\\GTISISTEMAS\\Banco\\GTI.FDB, read-only): a full read_evidence() scan took ~40ms for the whole
|
||||
catalog, and a targeted per-product lookup correctly drove field_contract.infer_role to RESALE/OWN_PRODUCTION for 5 of 8 real products with
|
||||
zero operator questions - this test locks in the behavior with a portable (sqlite) fixture."""
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
from gti_fiscal_agent.firebird_reader import read_product_evidence
|
||||
|
||||
|
||||
def _conn():
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE RECEITA_INGREDIENTES(CODRECEITA TEXT)")
|
||||
db.execute("CREATE TABLE C000032(CODPRODUTO TEXT, TIPO TEXT)")
|
||||
db.execute("CREATE TABLE C000088(CODPRODUTO TEXT, ICMS_RETIDO TEXT, ICMS_VALORRETIDO REAL, SUB_VALOR REAL)")
|
||||
db.execute("CREATE TABLE C000100(CODPRODUTO TEXT, ULTIMA_ENTRADA TEXT, NOTA_FISCAL TEXT, ENTRADAS REAL)")
|
||||
return db
|
||||
|
||||
|
||||
def test_no_firebird_system_table_degrades_to_no_evidence_never_raises():
|
||||
"""A non-Firebird connection (this project's own sqlite test fakes) has no RDB$RELATION_FIELDS: the capability probe must degrade
|
||||
gracefully to "no evidence", exactly like a missing table would, never raise."""
|
||||
db = sqlite3.connect(":memory:")
|
||||
ev = read_product_evidence(db, "000001")
|
||||
assert ev == {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
|
||||
|
||||
|
||||
def test_targeted_lookup_is_scoped_to_the_one_product_code():
|
||||
db = _conn()
|
||||
db.executemany("INSERT INTO RECEITA_INGREDIENTES VALUES (?)", [("000009",), ("000010",)])
|
||||
db.execute("INSERT INTO C000032 VALUES ('000011', 'COMPOSICAO')")
|
||||
db.execute("INSERT INTO C000088 VALUES ('000012', 'S', 0, 0)")
|
||||
db.execute("INSERT INTO C000100 VALUES ('000013', '2026-01-01', '000001', 5)")
|
||||
db.commit()
|
||||
|
||||
def has_relation_fields():
|
||||
cur = db.cursor()
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000032','C000088','C000100','RECEITA_INGREDIENTES')")
|
||||
return cur.fetchall()
|
||||
|
||||
# sqlite has no RDB$RELATION_FIELDS either; monkeypatch the cursor to answer the capability probe like a real Firebird database would
|
||||
class ProbedConn:
|
||||
def __init__(self, real):
|
||||
self.real = real
|
||||
|
||||
def cursor(self):
|
||||
return ProbedCursor(self.real.cursor())
|
||||
|
||||
class ProbedCursor:
|
||||
def __init__(self, real):
|
||||
self.real = real
|
||||
self._probe_result = None
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
if "RDB$RELATION_FIELDS" in sql:
|
||||
self._probe_result = [
|
||||
("RECEITA_INGREDIENTES", "CODRECEITA"), ("C000032", "CODPRODUTO"), ("C000032", "TIPO"),
|
||||
("C000088", "CODPRODUTO"), ("C000088", "ICMS_RETIDO"), ("C000088", "ICMS_VALORRETIDO"), ("C000088", "SUB_VALOR"),
|
||||
("C000100", "CODPRODUTO"), ("C000100", "ULTIMA_ENTRADA"), ("C000100", "NOTA_FISCAL"), ("C000100", "ENTRADAS"),
|
||||
]
|
||||
return
|
||||
self._probe_result = None
|
||||
sqlite_sql = sql.replace("SELECT FIRST 1 ", "SELECT ") + (" LIMIT 1" if "FIRST 1" in sql else "")
|
||||
self.real.execute(sqlite_sql, params)
|
||||
|
||||
def fetchall(self):
|
||||
return self._probe_result if self._probe_result is not None else self.real.fetchall()
|
||||
|
||||
def fetchone(self):
|
||||
return self.real.fetchone()
|
||||
|
||||
conn = ProbedConn(db)
|
||||
ev = read_product_evidence(conn, "000009")
|
||||
assert ev["recipes"] == {"000009"} and ev["composed"] == set() and ev["purchases"] == {} and ev["stock_entry"] == {} and ev["stock_in"] == {}
|
||||
ev = read_product_evidence(conn, "000011")
|
||||
assert ev["composed"] == {"000011"} and ev["recipes"] == set()
|
||||
ev = read_product_evidence(conn, "000012")
|
||||
assert ev["purchases"] == {"000012": [{"icms_retido": "S", "valor_retido": 0, "sub_valor": 0}]}
|
||||
ev = read_product_evidence(conn, "000013")
|
||||
assert ev["stock_entry"] == {"000013": True} and ev["stock_in"] == {"000013": True}
|
||||
ev = read_product_evidence(conn, "999999")
|
||||
assert ev == {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
|
||||
@@ -0,0 +1,52 @@
|
||||
from gti_fiscal_agent.ipi_auto_apply import ipi_write_decision
|
||||
|
||||
|
||||
def _decision(ipi_status="IPI_RESOLVED", status="EXACT_NCM", ipi=None):
|
||||
return {"status": status, "ipi_status": ipi_status, "classification": {"ipi": ipi}}
|
||||
|
||||
|
||||
def test_numeric_null_current_is_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), None)
|
||||
assert d["eligible"] is True
|
||||
assert d["proposed_value"] == 3.25
|
||||
assert d["reason_code"] == "CHANGE_REQUIRED"
|
||||
|
||||
|
||||
def test_numeric_same_value_is_not_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), 3.25)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "SAME_VALUE"
|
||||
|
||||
|
||||
def test_numeric_different_value_is_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), 0)
|
||||
assert d["eligible"] is True
|
||||
assert d["proposed_value"] == 3.25
|
||||
|
||||
|
||||
def test_non_numeric_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NON_NUMERIC", "rate_numeric": None}), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "RATE_KIND_NON_NUMERIC"
|
||||
|
||||
|
||||
def test_source_blank_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "BLANK", "rate_numeric": None}), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "RATE_KIND_BLANK"
|
||||
|
||||
|
||||
def test_ambiguous_ipi_status_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi_status="IPI_CANDIDATES", ipi=None), None)
|
||||
assert d["eligible"] is False
|
||||
|
||||
|
||||
def test_no_match_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi_status="NO_TIPI_MAPPING_FOUND", ipi=None), None)
|
||||
assert d["eligible"] is False
|
||||
|
||||
|
||||
def test_legacy_ncm_never_eligible():
|
||||
d = ipi_write_decision(_decision(status="LEGACY_NCM", ipi_status="LEGACY_NCM"), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "LEGACY_NCM"
|
||||
@@ -0,0 +1,132 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent.company_context import resolve
|
||||
from gti_fiscal_agent.preflight import major_minor, run_preflight
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
spec = importlib.util.spec_from_file_location("run_agent_service", ROOT / "agent" / "scripts" / "run_agent_service.py")
|
||||
service = importlib.util.module_from_spec(spec); sys.modules[spec.name] = service; spec.loader.exec_module(service)
|
||||
|
||||
CN = "11111111000111"
|
||||
|
||||
|
||||
def env(tmp_path, **over):
|
||||
base = {"GTI_HUB_URL": "http://hub.test", "AGENT_TOKEN": "SECRETTOKENVALUE9", "FIREBIRD_DATABASE_PATH": "x.fdb", "FB_USER": "u", "FB_PASSWORD": "SECRETPASSVALUE9", "FISCAL_MODE": "DRY_RUN", "FIREBIRD_WRITES": "0",
|
||||
"AGENT_STATE_DB": str(tmp_path / "state.sqlite"), "AGENT_LOG_PATH": str(tmp_path / "logs" / "a.log"), "AGENT_VERSION": "1.0.0-rc1"}
|
||||
base.update(over)
|
||||
return base
|
||||
|
||||
|
||||
def http_ok(url, token=None, timeout=10):
|
||||
if url.endswith("/health"):
|
||||
return 200, {"status": "ok"}
|
||||
if url.endswith("/version"):
|
||||
return 200, {"version": "1.0.0-rc1"}
|
||||
return (200 if token == "SECRETTOKENVALUE9" else 401), {}
|
||||
|
||||
|
||||
def fake_probe(company_over=None, caps_missing=()):
|
||||
def probe(env):
|
||||
cols = {"CLASSIFICACAO_FISCAL": (37, 20), "CFOP_ENTRADA": (14, 4), "CST_ENTRADA": (14, 4), "CFOP": (37, 4), "CST": (37, 10), "ALIQUOTA": (16, 8), "CEST": (37, 7), "SITUACAO_TRIBUTARIA": (37, 2),
|
||||
"COD_ENTRADA_COFINS": (37, 3), "COD_SAIDA_COFINS": (37, 3), "ALIQ_COFINS": (37, 8), "COD_ENTRADA_PIS": (37, 3), "COD_SAIDA_PIS": (37, 3), "ALIQ_PIS": (37, 8), "CSOSN": (37, 60), "IPI": (27, 8),
|
||||
"CST_IPI": (14, 3), "IAT": (37, 1), "INCIDENCIA_PISCOFINS": (37, 30), "PISCOFINS": (37, 1), "IPPT": (37, 1), "EX_TIPI": (37, 3), "COD_ORIGEM_MERCADORIA": (8, 4)}
|
||||
for c in caps_missing:
|
||||
cols.pop(c)
|
||||
ev = {"c000004": [{"code": "000001", "cnpj": CN, "uf": "BA", "crt": "1", "optante": "S"}], "emissor": [{"cnpj": CN, "uf": "BA", "crt": "1"}], "nfce_keys": [{"cuf": "29", "cnpj": CN, "n": 5}], "nf_by_filial": {}}
|
||||
ev.update(company_over or {})
|
||||
return {"server_version": "WI-V6.3.9 Firebird 2.5", "ods": "11.2", "tables": {"C000004": {}, "C000025": cols}, "products": 3712, "company": resolve(ev, {})}
|
||||
return probe
|
||||
|
||||
|
||||
def test_preflight_passes_for_a_clean_dry_run_installation(tmp_path):
|
||||
r = run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "PASS" and r["blockers"] == [] and r["effective_firebird_writes"] is False
|
||||
names = {c["name"] for c in r["checks"]}
|
||||
assert {"HUB_REACHABLE", "AUTH", "HUB_VERSION_COMPATIBLE", "FIREBIRD_REACHABLE", "C000004", "C000025", "CAPABILITIES", "COMPANY_CONTEXT", "UF", "CRT", "SCHEMA_TYPE_GUARDS", "SQLITE", "LOG_DIRECTORY", "FISCAL_MODE", "FIREBIRD_WRITES"} <= names
|
||||
assert "SECRETTOKENVALUE9" not in str(r) and "SECRETPASSVALUE9" not in str(r)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("over,blocker", [
|
||||
({"FISCAL_MODE": "DRY_RUN", "FIREBIRD_WRITES": "1"}, "FIREBIRD_WRITES"),
|
||||
({"FISCAL_MODE": ""}, "FISCAL_MODE"),
|
||||
({"AGENT_TOKEN": ""}, "CONFIG"),
|
||||
])
|
||||
def test_preflight_blocks_bad_config(tmp_path, over, blocker):
|
||||
r = run_preflight(env(tmp_path, **over), http_get=http_ok, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith(blocker) for b in r["blockers"])
|
||||
|
||||
|
||||
def test_preflight_fails_on_hub_auth_version_and_missing_capability(tmp_path):
|
||||
def bad_auth(url, token=None, timeout=10):
|
||||
return http_ok(url, "wrong" if token else None)
|
||||
assert any(b.startswith("AUTH") for b in run_preflight(env(tmp_path), http_get=bad_auth, probe=fake_probe())["blockers"])
|
||||
|
||||
def old_hub(url, token=None, timeout=10):
|
||||
return (200, {"version": "0.9.0"}) if url.endswith("/version") else http_ok(url, token)
|
||||
assert any(b.startswith("HUB_VERSION_COMPATIBLE") for b in run_preflight(env(tmp_path), http_get=old_hub, probe=fake_probe())["blockers"])
|
||||
assert any(b.startswith("CAPABILITIES") for b in run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(caps_missing=("CEST",)))["blockers"])
|
||||
|
||||
|
||||
def test_preflight_fails_when_company_context_is_ambiguous_or_crt_conflicts(tmp_path):
|
||||
two = {"c000004": [{"code": "1", "cnpj": CN, "uf": "BA", "crt": "1"}, {"code": "2", "cnpj": "22222222000122", "uf": "SP", "crt": "1"}], "emissor": []}
|
||||
r = run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(two))
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith("COMPANY_CONTEXT") for b in r["blockers"])
|
||||
conflict = {"emissor": [{"cnpj": CN, "uf": "BA", "crt": "3"}]}
|
||||
assert any("CRT_SOURCE_CONFLICT" in b for b in run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(conflict))["blockers"])
|
||||
|
||||
|
||||
def test_hub_unreachable_is_a_clear_blocker(tmp_path):
|
||||
def down(url, token=None, timeout=10):
|
||||
raise ConnectionError("nope")
|
||||
r = run_preflight(env(tmp_path), http_get=down, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith("HUB_REACHABLE") for b in r["blockers"])
|
||||
|
||||
|
||||
def test_major_minor():
|
||||
assert major_minor("1.0.0-rc1") == "1.0" and major_minor("1.0.3") == "1.0" and major_minor("0.9.0") != major_minor("1.0.0-rc1")
|
||||
|
||||
|
||||
# ---------------- zero-write guarantee ----------------
|
||||
def test_dry_run_makes_writes_technically_impossible_whatever_firebird_writes_says():
|
||||
for extra in ({"FIREBIRD_WRITES": "1", "FISCAL_MODE": "DRY_RUN"}, {"FIREBIRD_WRITES": "0", "FISCAL_MODE": "AUTO_APPLY"}, {"FIREBIRD_WRITES": "1", "FISCAL_MODE": "dry_run"}):
|
||||
cfg = service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", **extra})
|
||||
assert cfg.firebird_writes is False
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1", "FISCAL_MODE": "AUTO_APPLY"}).firebird_writes is True
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1"}).firebird_writes is True # legacy: unset FISCAL_MODE follows the flag
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb"}).fiscal_mode == "DRY_RUN"
|
||||
|
||||
|
||||
def test_write_connection_cannot_be_opened_when_writes_are_disabled(monkeypatch):
|
||||
opened = []
|
||||
monkeypatch.setitem(sys.modules, "fdb", type("F", (), {"connect": staticmethod(lambda **kw: opened.append(kw))}))
|
||||
with pytest.raises(RuntimeError):
|
||||
service.open_write_connection({"FB_USER": "u", "FB_PASSWORD": "SECRETPASSVALUE9"}, "x.fdb", firebird_writes=False)
|
||||
assert opened == []
|
||||
|
||||
|
||||
def test_dry_run_job_never_opens_a_write_connection_nor_calls_the_executor(monkeypatch):
|
||||
class Boom:
|
||||
def __getattr__(self, name):
|
||||
if name in ("execute_guarded_update", "execute_guarded_restore"):
|
||||
raise AssertionError("executor write called in DRY_RUN")
|
||||
return getattr(__import__("scripts.firebird_write_executor", fromlist=["x"]) if False else object(), name, None)
|
||||
cfg = service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1", "FISCAL_MODE": "DRY_RUN"})
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [{"CODIGO": "000001", "PRODUTO": "X", "CLASSIFICACAO_FISCAL": "22030000", "IPI": None}])
|
||||
monkeypatch.setattr(service, "HubClient", lambda url, token: type("H", (), {"classify_batch": lambda self, p: [{"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED", "classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.9}},
|
||||
"dataset_versions": {}}]})())
|
||||
monkeypatch.setattr(service, "build_job_coverage", lambda *a, **k: {"TOTAL_PRODUCTS": 1})
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda *a, **k: (_ for _ in ()).throw(AssertionError("write connection opened in DRY_RUN")))
|
||||
|
||||
class State:
|
||||
def log_ipi_write(self, *a): pass
|
||||
|
||||
class Ex:
|
||||
WriteRefused = Exception
|
||||
def execute_guarded_update(self, *a, **k): raise AssertionError("executor write called in DRY_RUN")
|
||||
|
||||
result, error = service.execute_job({"kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}, "job_id": 1}, cfg, Ex(), State(), {})
|
||||
assert error is None and result["writes_attempted"] == 0 and result["firebird_writes_enabled"] is False
|
||||
@@ -0,0 +1,444 @@
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from central.app import rtc_reference_data as ref
|
||||
from central.app.cclasstrib_resolver import local_catalog_hash
|
||||
from gti_fiscal_agent import rtc_auto_apply as apply_mod
|
||||
from gti_fiscal_agent import unified_fiscal as uf
|
||||
|
||||
pytestmark = pytest.mark.real_unified_fiscal
|
||||
|
||||
TODAY = date(2026, 9, 24)
|
||||
NORMAL = {"crt": "3"}
|
||||
CLASS_COLS = ("CCLASSTRIB", "CSTIBSCBS", "INICIO_VIGENCIA", "FIM_VIGENCIA", "TIPO_ALIQUOTA", "PRED_IBS", "PRED_CBS", "IND_NFE", "IND_NFCE")
|
||||
SCHEMA = ([("C000025", c) for c in ("CODIGO", "CCLASSTRIB", "CLASSIFICACAO_FISCAL", "PISCOFINS", "INCIDENCIA_PISCOFINS")] + [("RTC_CCLASSTRIB", c) for c in CLASS_COLS] + [("RTC_CST", "CSTIBSCBS"), ("PARAMETROS", "USAR_REFORMA_TRIBUTARIA")])
|
||||
|
||||
|
||||
class FakeFirebird:
|
||||
"""A tiny SGV: PARAMETROS, the RTC catalog (from the official table) and C000025. Every statement is recorded; only SELECT and the single allowed UPDATE are understood."""
|
||||
|
||||
def __init__(self, products, param=(1,), catalog_tweak=None, skip=()):
|
||||
self.products = {c: dict(v) for c, v in products.items()} # code -> {"CCLASSTRIB":..., "CLASSIFICACAO_FISCAL":...}
|
||||
self.param = list(param)
|
||||
self.catalog = {r["code"]: {"cst": r["cst"], "start": r["start"], "end": r["end"], "tipo": r["tipo_aliquota"], "pred_ibs": r["p_red_ibs"], "pred_cbs": r["p_red_cbs"], "ind_nfe": r["ind_nfe"], "ind_nfce": r["ind_nfce"]}
|
||||
for r in ref.OFFICIAL_CCLASSTRIB if r["code"] not in skip and not r["end"]}
|
||||
for code, change in (catalog_tweak or {}).items():
|
||||
self.catalog[code] = {**self.catalog[code], **change}
|
||||
self.cst_table = {r["cst"] for r in ref.OFFICIAL_CCLASSTRIB} # RTC_CST is its own table: a class may point to a CST that is not in it
|
||||
self.crt = ("3",) # C000004.CRT rows
|
||||
self.statements, self.commits, self.rollbacks = [], 0, 0
|
||||
self._pending = {}
|
||||
|
||||
def cursor(self):
|
||||
return FakeCursor(self)
|
||||
|
||||
def commit(self):
|
||||
for code, value in self._pending.items():
|
||||
self.products[code]["CCLASSTRIB"] = value
|
||||
self._pending = {}
|
||||
self.commits += 1
|
||||
|
||||
def rollback(self):
|
||||
self._pending = {}
|
||||
self.rollbacks += 1
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, db):
|
||||
self.db, self._rows, self.rowcount = db, [], -1
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
db = self.db
|
||||
db.statements.append((sql, tuple(params)))
|
||||
text = " ".join(sql.split())
|
||||
up = text.upper()
|
||||
if up.startswith("SELECT TRIM(RF.RDB$RELATION_NAME)"):
|
||||
self._rows = list(SCHEMA)
|
||||
elif up.startswith("SELECT FIRST 20 USAR_REFORMA_TRIBUTARIA"):
|
||||
self._rows = [(v,) for v in db.param]
|
||||
elif "RDB$RELATION_CONSTRAINTS" in up:
|
||||
self._rows = [(1,)]
|
||||
elif up.startswith("SELECT TRIM(CCLASSTRIB)") and "FROM RTC_CCLASSTRIB" in up:
|
||||
cols = [c.strip() for c in text[len("SELECT "):up.index(" FROM ")].split(",")]
|
||||
rows = []
|
||||
for code, r in db.catalog.items():
|
||||
mapping = {"CSTIBSCBS": r["cst"], "INICIO_VIGENCIA": date.fromisoformat(r["start"]) if r["start"] else None, "FIM_VIGENCIA": None, "TIPO_ALIQUOTA": r["tipo"], "PRED_IBS": r["pred_ibs"],
|
||||
"PRED_CBS": r["pred_cbs"], "IND_NFE": "1" if r["ind_nfe"] else "0", "IND_NFCE": "1" if r["ind_nfce"] else "0"}
|
||||
rows.append(tuple(code if c.startswith("TRIM(CCLASSTRIB") else mapping[re.sub(r"TRIM\((\w+)\)", r"\1", c)] for c in cols))
|
||||
self._rows = rows
|
||||
elif up.startswith("SELECT TRIM(CSTIBSCBS) FROM RTC_CST"):
|
||||
self._rows = [(c,) for c in sorted(db.cst_table)]
|
||||
elif up.startswith("SELECT CODIGO, CCLASSTRIB, CLASSIFICACAO_FISCAL"):
|
||||
self._rows = [(c, v["CCLASSTRIB"], v["CLASSIFICACAO_FISCAL"], "N", "NENHUMA") for c, v in sorted(db.products.items())]
|
||||
elif up.startswith("SELECT DISTINCT TRIM(CRT) FROM C000004"):
|
||||
self._rows = [(c,) for c in db.crt]
|
||||
elif up.startswith("SELECT CCLASSTRIB, CLASSIFICACAO_FISCAL, PRODUTO FROM C000025"):
|
||||
code = params[0]
|
||||
self._rows = [(db.products[code]["CCLASSTRIB"], db.products[code]["CLASSIFICACAO_FISCAL"], db.products[code].get("PRODUTO", ""))] if code in db.products else []
|
||||
if code in db.products and db.products[code].get("DUP"):
|
||||
self._rows = self._rows * 2
|
||||
elif up.startswith("SELECT CCLASSTRIB FROM C000025"):
|
||||
code = params[0]
|
||||
self._rows = [(db.products[code]["CCLASSTRIB"],)]
|
||||
elif up.startswith("UPDATE C000025 SET CCLASSTRIB = NULL WHERE CODIGO = ? AND TRIM(CCLASSTRIB) = ?"):
|
||||
code, written = params
|
||||
product = db.products.get(code)
|
||||
hit = product is not None and (product["CCLASSTRIB"] or "").strip() == written
|
||||
self.rowcount = 1 if hit else 0
|
||||
if hit:
|
||||
db._pending[code] = None
|
||||
elif up.startswith("UPDATE C000025 SET CCLASSTRIB = ? WHERE CODIGO = ?"):
|
||||
value, code = params
|
||||
product = db.products.get(code)
|
||||
hit = product is not None and not (product["CCLASSTRIB"] or "").strip()
|
||||
self.rowcount = (2 if product and product.get("DUP") else 1) if hit else 0
|
||||
if hit and self.rowcount == 1:
|
||||
db._pending[code] = value
|
||||
else:
|
||||
raise AssertionError(f"unexpected statement: {sql}")
|
||||
|
||||
def fetchall(self):
|
||||
return self._rows
|
||||
|
||||
def fetchone(self):
|
||||
return self._rows[0] if self._rows else None
|
||||
|
||||
|
||||
def product(ncm, value=None, **extra):
|
||||
return {"CCLASSTRIB": value, "CLASSIFICACAO_FISCAL": ncm, **extra}
|
||||
|
||||
|
||||
def assess(db, ncm_ok=None, company=NORMAL):
|
||||
return uf.assess_from_connection(db, company, today=TODAY, ncm_valid=ncm_ok if ncm_ok is not None else {c: True for c in db.products})
|
||||
|
||||
|
||||
def candidates_of(db, limit=500):
|
||||
out = assess(db)
|
||||
products = [{"CODIGO": c, "CCLASSTRIB": v["CCLASSTRIB"], "CLASSIFICACAO_FISCAL": v["CLASSIFICACAO_FISCAL"]} for c, v in db.products.items()]
|
||||
cands, total = apply_mod.select_candidates(out["findings"], products, limit=limit)
|
||||
return out, cands, total
|
||||
|
||||
|
||||
def run(db, cands, out, **kw):
|
||||
return apply_mod.apply_candidates(db, cands, company=NORMAL, decision={"local_catalog_hash": out["context"]["local_catalog_hash"]}, now="2026-09-24T20:00:00+00:00", job_id=7, today=TODAY, **kw)
|
||||
|
||||
|
||||
PRODUCTS = {"000001": product("10063021"), "000002": product("04090000"), "000003": product("15171000"), "000004": product("30049099"), "000005": product("39235000", "000001")}
|
||||
|
||||
|
||||
def test_gate_is_driven_by_the_central_mode_only_and_defaults_to_closed():
|
||||
assert apply_mod.write_gate(mode="ENABLED")["allowed"] is True
|
||||
assert apply_mod.write_gate(mode="APPROVAL_REQUIRED", approved=True)["allowed"] is True
|
||||
for kwargs in ({"mode": "DISABLED"}, {"mode": "DRY_RUN"}, {"mode": "APPROVAL_REQUIRED", "approved": False}, {"mode": "ENABLED", "capability": False}, {"mode": "bogus"}, {"mode": ""}):
|
||||
g = apply_mod.write_gate(**kwargs)
|
||||
assert g["allowed"] is False and g["missing"]
|
||||
assert apply_mod.ALLOWED_RTC_WRITE_FIELDS == ("CCLASSTRIB",) and apply_mod.CAPABILITY == "RTC_CCLASSTRIB_WRITE_V1"
|
||||
|
||||
|
||||
def test_only_proven_products_become_candidates():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, total = candidates_of(db)
|
||||
assert sorted((c["product_code"], c["proposed"], c["rule_id"]) for c in cands) == [("000001", "200003", "RTC-R3"), ("000002", "200034", "RTC-R3"), ("000003", "200003", "RTC-R3")]
|
||||
assert total == 3 and out["context"]["rtc_resolution"]["ready_to_apply"] == 3 and out["context"]["rtc_resolution"]["ambiguous"] == 1 # medicine chapter is ambiguous
|
||||
assert out["context"]["rtc_resolution"]["valid_existing"] == 1 # 000005 already has the right class
|
||||
assert "000004" not in {c["product_code"] for c in cands}
|
||||
|
||||
|
||||
def test_apply_writes_only_cclasstrib_with_audit_and_second_run_writes_nothing():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
result = run(db, cands, out)
|
||||
assert result["write_successes"] == 3 and result["write_attempts"] == 3 and result["aborted"] == []
|
||||
assert {c: db.products[c]["CCLASSTRIB"] for c in ("000001", "000002", "000003")} == {"000001": "200003", "000002": "200034", "000003": "200003"}
|
||||
updates = [s for s, _ in db.statements if s.upper().startswith("UPDATE")]
|
||||
assert len(updates) == 3 and all(re.fullmatch(r"UPDATE C000025 SET CCLASSTRIB = \? WHERE CODIGO = \? AND \(CCLASSTRIB IS NULL OR TRIM\(CCLASSTRIB\) = ''\)", u) for u in updates)
|
||||
assert not [s for s, _ in db.statements if re.match(r"(INSERT|DELETE|ALTER|DROP)", s.strip(), re.I)]
|
||||
audit = result["applied"][0]
|
||||
assert {"product_code", "field", "old_value", "new_value", "rule_id", "evidence", "official_source_version", "local_catalog_hash", "timestamp", "job_id", "outcome"} <= set(audit)
|
||||
assert audit["old_value"] is None and audit["field"] == "CCLASSTRIB" and audit["official_source_version"]["cclasstrib_sha256"] == ref.OFFICIAL_CATALOG["sha256"] and audit["job_id"] == 7
|
||||
out2, cands2, total2 = candidates_of(db) # idempotent: nothing left to write
|
||||
assert cands2 == [] and total2 == 0 and out2["context"]["rtc_resolution"]["valid_existing"] == 4
|
||||
before = len(db.statements)
|
||||
assert run(db, cands2, out2)["write_attempts"] == 0 and len(db.statements) == before
|
||||
|
||||
|
||||
def test_rtc_switched_off_between_decision_and_write_aborts_every_product():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
db.param = [0]
|
||||
result = run(db, cands, out)
|
||||
assert result["write_successes"] == 0 and result["aborted_reasons"] == {"RTC_NO_LONGER_ENABLED": 3} and all(p["CCLASSTRIB"] in (None, "000001") for p in db.products.values())
|
||||
|
||||
|
||||
def test_conflicting_parameter_rows_abort():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
db.param = [1, 0]
|
||||
assert run(db, cands, out)["aborted_reasons"] == {"RTC_NO_LONGER_ENABLED": 3}
|
||||
|
||||
|
||||
def test_catalog_changed_or_invalid_between_decision_and_write_aborts():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
db.catalog["200003"]["pred_ibs"] = "60"
|
||||
assert run(db, cands, out)["aborted_reasons"] == {"LOCAL_CATALOG_CHANGED_SINCE_DECISION": 3}
|
||||
db2 = FakeFirebird(PRODUCTS)
|
||||
out2, cands2, _ = candidates_of(db2)
|
||||
db2.catalog["999999"] = {**db2.catalog["000001"], "cst": "555"} # class pointing to a CST that does not exist -> invalid catalog
|
||||
invalid = apply_mod.apply_candidates(db2, cands2, company=NORMAL, decision={}, now="2026-09-24T20:00:00+00:00", job_id=7, today=TODAY) # no hash pinned: integrity check alone must stop it
|
||||
assert invalid["aborted_reasons"] == {"RTC_CATALOG_INVALID": 3}
|
||||
|
||||
|
||||
def test_product_changed_between_decision_and_write_is_left_alone():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
db.products["000001"]["CLASSIFICACAO_FISCAL"] = "30049099" # NCM edited in the ERP meanwhile
|
||||
db.products["000002"]["CCLASSTRIB"] = "200034" # operator filled it meanwhile (same value)
|
||||
db.products["000003"]["CCLASSTRIB"] = "200034" # ... or with a DIFFERENT value: never overwritten
|
||||
result = run(db, cands, out)
|
||||
assert result["aborted_reasons"] == {"PRODUCT_CHANGED_NCM": 1, "NO_WRITE_REQUIRED": 1, "VALUE_ALREADY_SET_NEVER_OVERWRITTEN": 1} and result["write_successes"] == 0
|
||||
assert db.products["000003"]["CCLASSTRIB"] == "200034" and not [s for s, _ in db.statements if s.upper().startswith("UPDATE")]
|
||||
|
||||
|
||||
def test_duplicate_code_rowcount_protection_and_isolation_of_failures():
|
||||
products = {**PRODUCTS, "000002": product("04090000", DUP=True)}
|
||||
db = FakeFirebird(products)
|
||||
out, cands, _ = candidates_of(db)
|
||||
result = run(db, cands, out)
|
||||
assert result["aborted_reasons"] == {"PRODUCT_ROW_NOT_UNIQUE": 1} and result["write_successes"] == 2 # the ambiguous row is skipped, the rest still applied
|
||||
assert db.products["000002"]["CCLASSTRIB"] is None
|
||||
|
||||
|
||||
def test_existing_different_valid_class_is_a_conflict_blocker_never_a_write():
|
||||
db = FakeFirebird({"000001": product("10063021", "000001")}) # rice already classified as fully taxed: differs from the proven Anexo I class
|
||||
out = assess(db)
|
||||
types = {b["type"] for b in out["findings"]["000001"]["blockers"]}
|
||||
assert types == {"CCLASSTRIB_CONFLICT"} and out["context"]["rtc_resolution"]["conflicts"] == 1
|
||||
_, cands, _ = candidates_of(db)
|
||||
assert cands == []
|
||||
|
||||
|
||||
def test_official_mismatch_of_an_existing_class_is_reported():
|
||||
db = FakeFirebird({"000001": product("39235000", "000001")}, catalog_tweak={"000001": {"ind_nfe": False, "ind_nfce": False}})
|
||||
types = {b["type"] for b in assess(db)["findings"]["000001"]["blockers"]}
|
||||
assert types == {"CCLASSTRIB_OFFICIAL_MISMATCH"}
|
||||
|
||||
|
||||
def test_batch_limit_bounds_the_blast_radius():
|
||||
db = FakeFirebird({f"{i:06d}": product("04090000") for i in range(1, 8)})
|
||||
out, cands, total = candidates_of(db, limit=3)
|
||||
assert len(cands) == 3 and total == 7
|
||||
|
||||
|
||||
def test_simples_never_produces_candidates_even_with_rtc_on():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out = uf.assess_from_connection(db, {"crt": "1"}, today=TODAY, ncm_valid={c: True for c in PRODUCTS})
|
||||
products = [{"CODIGO": c, "CCLASSTRIB": v["CCLASSTRIB"], "CLASSIFICACAO_FISCAL": v["CLASSIFICACAO_FISCAL"]} for c, v in db.products.items()]
|
||||
candidates, total = apply_mod.select_candidates(out["findings"], products)
|
||||
|
||||
assert out["context"]["rtc_resolution"]["ready_to_apply"] == 0
|
||||
assert out["context"]["rtc_resolution"]["not_applicable"] == len(PRODUCTS)
|
||||
assert candidates == [] and total == 0
|
||||
assert not [b for f in out["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert all(sql.lstrip().upper().startswith("SELECT") for sql, _ in db.statements)
|
||||
|
||||
|
||||
def test_rtc_off_has_no_resolution_and_no_candidates():
|
||||
db = FakeFirebird(PRODUCTS, param=(0,))
|
||||
out, cands, total = candidates_of(db)
|
||||
assert cands == [] and total == 0 and out["context"]["rtc_resolution"] is None
|
||||
|
||||
|
||||
def test_catalog_hash_is_recorded_for_audit():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out = assess(db)
|
||||
schema, catalog, _ = uf.load_rtc_context(db.cursor())
|
||||
assert out["context"]["local_catalog_hash"] == local_catalog_hash(catalog["rows"]) and re.fullmatch(r"[0-9a-f]{64}", out["context"]["local_catalog_hash"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- job integration (gates)
|
||||
import pytest # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _hub_echoes_the_job_authorization(monkeypatch):
|
||||
"""By default the Hub still says what the job said; individual tests change that to prove the pre-write re-check."""
|
||||
from test_run_agent_service import service
|
||||
monkeypatch.setattr(service, "_fetch_authorizations", lambda config, job: dict(job.get("authorizations") or {}))
|
||||
|
||||
|
||||
def _service():
|
||||
from test_run_agent_service import _config, service
|
||||
return service, _config
|
||||
|
||||
|
||||
def _fiscal_for(db):
|
||||
return assess(db)
|
||||
|
||||
|
||||
def _products_of(db):
|
||||
return [{"CODIGO": c, "CCLASSTRIB": v["CCLASSTRIB"], "CLASSIFICACAO_FISCAL": v["CLASSIFICACAO_FISCAL"], "PRODUTO": v.get("PRODUTO", "")} for c, v in db.products.items()]
|
||||
|
||||
|
||||
def _no_write(service, monkeypatch):
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda *a, **k: (_ for _ in ()).throw(AssertionError("no write connection may be opened")))
|
||||
|
||||
|
||||
def test_job_without_a_central_mode_or_with_disabled_or_dry_run_never_writes(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
_no_write(service, monkeypatch)
|
||||
for auth in ({}, {"rtc_write_mode": "DISABLED"}, {"rtc_write_mode": "DRY_RUN"}, {"rtc_write_mode": "bogus"}, {"rtc_write_mode": "APPROVAL_REQUIRED", "rtc_approved_codes": []}, {"rtc_auto_apply": True}):
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": auth}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["gate"]["allowed"] is False and summary["ready_total"] == 3 and summary["write_attempts"] == 0 and summary["applied"] == []
|
||||
off = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "DISABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
dry = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "DRY_RUN"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert off["preview"] == [] and off["mode"] == "REPORT_ONLY"
|
||||
assert [c["product_code"] for c in dry["preview"]] == ["000001", "000002", "000003"] and all(c["rule_id"] == "RTC-R3" for c in dry["preview"]) and dry["capability"] == "RTC_CCLASSTRIB_WRITE_V1"
|
||||
|
||||
|
||||
def test_r4_suggestions_are_never_in_the_preview_and_never_written(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird({"000001": product("39235000"), "000002": product("39235000")})
|
||||
fiscal = _fiscal_for(db)
|
||||
_no_write(service, monkeypatch)
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["ready_total"] == 0 and summary["preview"] == [] and summary["write_attempts"] == 0
|
||||
|
||||
|
||||
def test_enabled_mode_applies_without_any_local_switch(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: _Closeable(db))
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 9, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL) # default config: no firebird_writes, no env switch
|
||||
assert summary["mode"] == "ENABLED" and summary["write_successes"] == 3 and summary["allowed_fields"] == ["CCLASSTRIB"] and all(a["job_id"] == 9 and a["authorization_origin"] == "HUB_MODE:ENABLED" for a in summary["applied"])
|
||||
assert [db.products[c]["CCLASSTRIB"] for c in ("000001", "000002", "000003")] == ["200003", "200034", "200003"]
|
||||
a = summary["applied"][0]
|
||||
assert a["rule_version"] and a["company_crt"] == "3" and a["action"] == "APPLY" and a["rtc_status"] == "ENABLED" and a["local_catalog_state"] == "MATCH_SAFE" and a["old_value"] is None
|
||||
|
||||
|
||||
def test_approval_required_writes_only_the_approved_products(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: _Closeable(db))
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 9, "authorizations": {"rtc_write_mode": "APPROVAL_REQUIRED", "rtc_approved_codes": ["000002"]}}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["write_successes"] == 1 and [db.products[c]["CCLASSTRIB"] for c in ("000001", "000002", "000003")] == [None, "200034", None]
|
||||
|
||||
|
||||
def test_job_never_writes_when_the_erp_says_rtc_is_off_even_in_enabled_mode(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS, param=(0,))
|
||||
fiscal = _fiscal_for(db)
|
||||
_no_write(service, monkeypatch)
|
||||
products = [{"CODIGO": c, "CCLASSTRIB": None, "CLASSIFICACAO_FISCAL": "04090000"} for c in db.products]
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "ENABLED"}}, products, fiscal, NORMAL)
|
||||
assert summary["write_attempts"] == 0 and summary["ready_total"] == 0
|
||||
|
||||
|
||||
def test_company_regime_or_fingerprint_change_aborts_before_the_update():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
db.crt = ("1",) # the company was switched to Simples between decision and write
|
||||
res = run(db, cands, out)
|
||||
assert res["write_successes"] == 0 and res["aborted_reasons"] == {"COMPANY_CONTEXT_CHANGED": 3}
|
||||
db.crt = ("3", "1") # ambiguous company context is also refused
|
||||
assert run(db, cands, out)["aborted_reasons"] == {"COMPANY_CONTEXT_CHANGED": 3}
|
||||
db.crt = ("3",)
|
||||
db.products["000001"]["PRODUTO"] = "RENAMED AFTER THE DECISION"
|
||||
res = run(db, cands, out)
|
||||
assert res["aborted_reasons"] == {"PRODUCT_CHANGED_SINCE_DECISION": 1} and res["write_successes"] == 2 and db.products["000001"]["CCLASSTRIB"] is None
|
||||
assert not any("UPDATE" in q.upper() and "C000004" in q.upper() for q, _ in db.statements)
|
||||
|
||||
|
||||
def test_rollback_restores_only_when_the_value_still_equals_the_written_one_and_reapply_works():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
first = run(db, cands, out)
|
||||
assert first["write_successes"] == 3
|
||||
items = [{"audit_id": i + 1, "product_code": a["product_code"], "written_value": a["new_value"], "rule_id": a["rule_id"]} for i, a in enumerate(first["applied"])]
|
||||
db.products["000002"]["CCLASSTRIB"] = "200035" # an operator changed it afterwards
|
||||
res = apply_mod.rollback_records(db, items, now="2026-09-24T21:00:00+00:00", job_id=8)
|
||||
by = {r["product_code"]: r for r in res["rollbacks"]}
|
||||
assert res["rolled_back"] == 2 and by["000002"]["outcome"] == "ABORTED" and by["000002"]["reason"] == "ROLLBACK_CONFLICT_VALUE_CHANGED" and db.products["000002"]["CCLASSTRIB"] == "200035"
|
||||
assert by["000001"]["outcome"] == "ROLLED_BACK" and by["000001"]["rollback_of"] == 1 and by["000001"]["old_value"] == "200003" and by["000001"]["new_value"] is None and db.products["000001"]["CCLASSTRIB"] is None
|
||||
again = apply_mod.rollback_records(db, items, now="2026-09-24T21:05:00+00:00", job_id=9)["rollbacks"]
|
||||
assert [r["reason"] for r in again if r["product_code"] == "000001"] == ["ALREADY_EMPTY"]
|
||||
out2, cands2, _ = candidates_of(db) # reapply: the emptied products are candidates again, the changed one is not
|
||||
assert sorted(c["product_code"] for c in cands2) == ["000001", "000003"] and run(db, cands2, out2)["write_successes"] == 2
|
||||
|
||||
|
||||
def test_rollback_uses_one_fixed_statement_and_rejects_malformed_requests():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
bad = apply_mod.rollback_records(db, [{"audit_id": 1, "product_code": "000001", "written_value": ""}, {"audit_id": 2, "product_code": "000001", "written_value": "200003", "restore_value": "999999"}], now="t")
|
||||
assert [r["reason"] for r in bad["rollbacks"]] == ["ROLLBACK_REQUEST_INVALID"] * 2 and db.statements == []
|
||||
db.products["000001"]["CCLASSTRIB"] = "200003"
|
||||
apply_mod.rollback_records(db, [{"audit_id": 3, "product_code": "000001", "written_value": "200003"}], now="t")
|
||||
writes = [q for q, _ in db.statements if q.strip().upper().startswith(("UPDATE", "INSERT", "DELETE"))]
|
||||
assert writes == ["UPDATE C000025 SET CCLASSTRIB = NULL WHERE CODIGO = ? AND TRIM(CCLASSTRIB) = ?"]
|
||||
|
||||
|
||||
class _Closeable:
|
||||
def __init__(self, db):
|
||||
self._db = db
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._db, name)
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_firebird_unavailable_for_the_write_connection_fails_the_job_closed_with_zero_writes(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda *a, **k: (_ for _ in ()).throw(OSError("Firebird unavailable")))
|
||||
import pytest as _pytest
|
||||
with _pytest.raises(OSError):
|
||||
service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert all(v["CCLASSTRIB"] is None for c, v in db.products.items() if c != "000005") and not [q for q, _ in db.statements if q.upper().startswith("UPDATE")]
|
||||
|
||||
|
||||
def test_connection_is_closed_even_when_a_product_fails_mid_batch(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird({**PRODUCTS, "000002": product("04090000", DUP=True)})
|
||||
fiscal = _fiscal_for(db)
|
||||
closed = []
|
||||
conn = _Closeable(db)
|
||||
conn.close = lambda: closed.append(True)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: conn)
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert closed == [True] and summary["write_successes"] == 2 and summary["aborted_reasons"]
|
||||
|
||||
|
||||
def test_authorization_revoked_after_the_job_was_claimed_means_zero_writes(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: _Closeable(db))
|
||||
monkeypatch.setattr(service, "_fetch_authorizations", lambda config, job: {"rtc_write_mode": "DISABLED"}) # operator switched it off while the job was running
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 5, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["write_successes"] == 0 and summary["aborted_reasons"] == {"AUTHORIZATION_CHANGED": 3}
|
||||
assert [db.products[c]["CCLASSTRIB"] for c in ("000001", "000002", "000003")] == [None, None, None]
|
||||
|
||||
|
||||
def test_unreachable_hub_is_not_authorization_and_approval_removal_is_honoured(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: _Closeable(db))
|
||||
|
||||
def down(config, job):
|
||||
raise ConnectionError("hub down")
|
||||
|
||||
monkeypatch.setattr(service, "_fetch_authorizations", down)
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 6, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["write_successes"] == 0 and summary["aborted_reasons"] == {"AUTHORIZATION_UNVERIFIABLE": 3}
|
||||
monkeypatch.setattr(service, "_fetch_authorizations", lambda config, job: {"rtc_write_mode": "APPROVAL_REQUIRED", "rtc_approved_codes": ["000001"]}) # 000002 was un-approved meanwhile
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 7, "authorizations": {"rtc_write_mode": "APPROVAL_REQUIRED", "rtc_approved_codes": ["000001", "000002"]}}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["write_successes"] == 1 and db.products["000001"]["CCLASSTRIB"] == "200003" and db.products["000002"]["CCLASSTRIB"] is None
|
||||
@@ -0,0 +1,515 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT = Path(__file__).parents[1] / "scripts" / "run_agent_service.py"
|
||||
AUTHORIZED = r"D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB"
|
||||
|
||||
|
||||
def load_module():
|
||||
spec = importlib.util.spec_from_file_location("run_agent_service", SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
service = load_module()
|
||||
|
||||
|
||||
class _FakeState:
|
||||
def __init__(self):
|
||||
self.logged = []
|
||||
self.identity = None
|
||||
|
||||
def stored_installation_id(self):
|
||||
return self.identity
|
||||
|
||||
def installation_id(self, candidate=None):
|
||||
if self.identity and candidate and self.identity != candidate:
|
||||
raise ValueError("configured installation_id conflicts with persisted identity")
|
||||
if self.identity is None:
|
||||
if not candidate:
|
||||
raise ValueError("installation_id is required")
|
||||
self.identity = candidate
|
||||
return self.identity
|
||||
|
||||
def log_ipi_write(self, *args):
|
||||
self.logged.append(args)
|
||||
|
||||
|
||||
class _FakeExecutor:
|
||||
"""Mirrors the real firebird_write_executor's public surface used by execute_job."""
|
||||
class WriteRefused(Exception):
|
||||
pass
|
||||
|
||||
def __init__(self, path_guard_ok=True, updates=None, fingerprint_guard_ok=True):
|
||||
self.path_guard_ok = path_guard_ok
|
||||
self.updates = updates or {}
|
||||
self.fingerprint_guard_ok = fingerprint_guard_ok
|
||||
|
||||
def assert_authorized_write_path(self, path, authorized_path=None):
|
||||
if not self.path_guard_ok:
|
||||
raise self.WriteRefused(f"path is not the authorized write target: {path}")
|
||||
|
||||
def assert_schema_fingerprint_authorized(self, observed, expected):
|
||||
if not self.fingerprint_guard_ok or observed != expected:
|
||||
raise self.WriteRefused(f"schema fingerprint mismatch: expected {expected!r}, observed {observed!r}")
|
||||
|
||||
def execute_guarded_update(self, connection, *, database_path, proposal, now, authorized_path=None):
|
||||
code = proposal["product_code"]
|
||||
if code in self.updates:
|
||||
raise self.WriteRefused(self.updates[code])
|
||||
record = type("Record", (), {
|
||||
"before_value": proposal["current_value"],
|
||||
"after_value": proposal["proposed_value"],
|
||||
"rowcount": 1,
|
||||
})()
|
||||
return record
|
||||
|
||||
|
||||
def _config(**overrides):
|
||||
base = dict(
|
||||
hub_url="http://localhost:8000", agent_id="a1", agent_token="tok", company_id="c1", company_uf="BA",
|
||||
database_path=AUTHORIZED, firebird_writes=False, poll_interval_seconds=0.01,
|
||||
agent_version="0.1.0", schema_fingerprint="fp", state_db=":memory:", agent_jobs_enabled=True,
|
||||
authorized_write_database_path=None, expected_schema_fingerprint="fp",
|
||||
)
|
||||
base.update(overrides)
|
||||
return type("Config", (), base)()
|
||||
|
||||
|
||||
def test_config_defaults_from_env():
|
||||
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED}
|
||||
config = service.Config(env)
|
||||
assert config.firebird_writes is False
|
||||
assert config.hub_url == "http://localhost:8000"
|
||||
assert config.poll_interval_seconds == 5.0
|
||||
assert config.agent_jobs_enabled is True
|
||||
|
||||
|
||||
def test_local_agent_jobs_toggle_is_ignored_control_is_central():
|
||||
# Zero-touch: a stale AGENT_JOBS_ENABLED=0 left in an old agent.env must not silence the Agent;
|
||||
# only the Hub (heartbeat processing_enabled) can pause it.
|
||||
config = service.Config({"FIREBIRD_DATABASE_PATH": AUTHORIZED, "AGENT_JOBS_ENABLED": "0"})
|
||||
|
||||
assert config.agent_jobs_enabled is True
|
||||
|
||||
|
||||
def test_config_reads_gti_hub_url_and_writes_flag():
|
||||
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "GTI_HUB_URL": "http://hub.internal", "FIREBIRD_WRITES": "1"}
|
||||
config = service.Config(env)
|
||||
assert config.hub_url == "http://hub.internal"
|
||||
assert config.firebird_writes is True
|
||||
|
||||
|
||||
def test_identity_recovery_persists_existing_hub_installation_id(tmp_path):
|
||||
config = _config(agent_id="2")
|
||||
state = service.AgentState(tmp_path / "agent.db")
|
||||
installation_id = "00000000-0000-4000-8000-000000000222"
|
||||
|
||||
class FakeIdentityClient:
|
||||
calls = 0
|
||||
|
||||
def identity(self):
|
||||
self.calls += 1
|
||||
return {"agent_id": 2, "installation_id": installation_id}
|
||||
|
||||
client = FakeIdentityClient()
|
||||
recovered = service.resolve_installation_id(config, {}, state, client)
|
||||
|
||||
assert recovered == installation_id
|
||||
assert state.installation_id() == installation_id
|
||||
assert client.calls == 1
|
||||
|
||||
restarted = service.resolve_installation_id(config, {}, state, client)
|
||||
assert restarted == installation_id
|
||||
assert client.calls == 1
|
||||
|
||||
|
||||
def test_identity_recovery_fails_closed_for_another_agent_or_missing_id(tmp_path):
|
||||
config = _config(agent_id="2")
|
||||
|
||||
class FakeIdentityClient:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
def identity(self):
|
||||
return self.payload
|
||||
|
||||
state = service.AgentState(tmp_path / "agent.db")
|
||||
with pytest.raises(RuntimeError, match="identity does not match configured Agent"):
|
||||
service.resolve_installation_id(
|
||||
config,
|
||||
{},
|
||||
state,
|
||||
FakeIdentityClient({"agent_id": 3, "installation_id": "00000000-0000-4000-8000-000000000333"}),
|
||||
)
|
||||
assert state.connection.execute("SELECT COUNT(*) FROM installation_identity").fetchone()[0] == 0
|
||||
|
||||
with pytest.raises(RuntimeError, match="identity response has no installation_id"):
|
||||
service.resolve_installation_id(config, {}, state, FakeIdentityClient({"agent_id": 2}))
|
||||
assert state.connection.execute("SELECT COUNT(*) FROM installation_identity").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_path_guard_fails_closed_before_any_write(monkeypatch):
|
||||
config = _config(firebird_writes=True, database_path=r"D:\PRODUCAO\ERP.FDB")
|
||||
executor = _FakeExecutor(path_guard_ok=False)
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
called = {"read_products": False}
|
||||
|
||||
def fake_read_products(env, codigos=None):
|
||||
called["read_products"] = True
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(service, "read_products", fake_read_products)
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "not the authorized write target" in error
|
||||
assert called["read_products"] is False # never even reads Firebird once the guard refuses
|
||||
|
||||
|
||||
def test_pilot_write_blocked_when_schema_fingerprint_does_not_match(monkeypatch):
|
||||
# A real client install where the live schema fingerprint drifted from what
|
||||
# this Agent installation was configured/registered to expect.
|
||||
config = _config(firebird_writes=True, schema_fingerprint="observed-live-fp", expected_schema_fingerprint="registered-fp")
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
called = {"read_products": False}
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: called.update(read_products=True) or [])
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "schema fingerprint mismatch" in error
|
||||
assert called["read_products"] is False
|
||||
|
||||
|
||||
def test_pilot_write_allowed_with_explicit_authorized_path_and_matching_fingerprint(monkeypatch):
|
||||
client_path = r"D:\ClienteX\ERP\PRODUCAO.FDB"
|
||||
config = _config(
|
||||
firebird_writes=True, database_path=client_path, authorized_write_database_path=client_path,
|
||||
schema_fingerprint="fp", expected_schema_fingerprint="fp",
|
||||
)
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{
|
||||
"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED",
|
||||
"classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}},
|
||||
"dataset_versions": {"tipi_ipi": "v1"},
|
||||
}]
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
fake_connection = type("Conn", (), {"close": lambda self: None})()
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: fake_connection)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert result["writes_succeeded"] == 1
|
||||
|
||||
|
||||
def test_execute_job_isolates_internal_errors_without_raising(monkeypatch):
|
||||
config = _config()
|
||||
|
||||
def boom(env, codigos=None):
|
||||
raise RuntimeError("firebird unavailable")
|
||||
|
||||
monkeypatch.setattr(service, "read_products", boom)
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "firebird unavailable" in error
|
||||
|
||||
|
||||
def test_execute_job_success_summarizes_pipeline(monkeypatch):
|
||||
config = _config()
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{
|
||||
"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED",
|
||||
"classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}},
|
||||
"dataset_versions": {"tipi_ipi": "v1"},
|
||||
}]
|
||||
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert result["products_processed"] == 1
|
||||
assert result["writes_attempted"] == 0 # firebird_writes disabled by default in _config()
|
||||
|
||||
|
||||
def test_reprocess_products_resolves_real_company_context_not_none(monkeypatch):
|
||||
"""REPROCESS_PRODUCTS used to hardcode company=None for _unified_fiscal (it isn't in SYNC_JOB_KINDS). Company resolution is cheap and
|
||||
deterministic (company_context.resolve, same as every sync job already uses) - it must never silently fall back to None when the company
|
||||
genuinely resolves."""
|
||||
config = _config()
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED", "classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}}, "dataset_versions": {"tipi_ipi": "v1"}}]
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
monkeypatch.setattr(service._firebird_reader, "read_company_context", lambda env: {"uf": "BA", "crt": "1", "company_code": "000001"})
|
||||
|
||||
captured = {}
|
||||
real_unified = service._unified_fiscal
|
||||
|
||||
def spy_unified_fiscal(env, company, ncm_valid=None):
|
||||
captured["company"] = company
|
||||
return real_unified(env, company, ncm_valid)
|
||||
monkeypatch.setattr(service, "_unified_fiscal", spy_unified_fiscal)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert captured["company"] == {"crt": "1"} # real CRT, never None, when the company genuinely resolves
|
||||
|
||||
|
||||
def test_reprocess_products_falls_back_to_none_only_when_company_unresolved(monkeypatch):
|
||||
"""No guessing: when the company genuinely cannot be resolved (e.g. Firebird unreachable), company stays None - same as before."""
|
||||
config = _config()
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED", "classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}}, "dataset_versions": {"tipi_ipi": "v1"}}]
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
|
||||
def boom(env):
|
||||
raise RuntimeError("firebird unavailable")
|
||||
monkeypatch.setattr(service._firebird_reader, "read_company_context", boom)
|
||||
|
||||
captured = {}
|
||||
real_unified = service._unified_fiscal
|
||||
|
||||
def spy_unified_fiscal(env, company, ncm_valid=None):
|
||||
captured["company"] = company
|
||||
return real_unified(env, company, ncm_valid)
|
||||
monkeypatch.setattr(service, "_unified_fiscal", spy_unified_fiscal)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert captured["company"] is None
|
||||
|
||||
|
||||
class _FakeHubForLoop:
|
||||
"""Simulates: heartbeat flaky twice then OK, one job then nothing, and captures completions."""
|
||||
|
||||
def __init__(self):
|
||||
self.heartbeat_calls = 0
|
||||
self.jobs = [{"job_id": 1, "kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}}]
|
||||
self.completed = []
|
||||
self.failed = []
|
||||
|
||||
def heartbeat(self, *a, **k):
|
||||
self.heartbeat_calls += 1
|
||||
if self.heartbeat_calls == 1:
|
||||
raise ConnectionError("hub down")
|
||||
return {}
|
||||
|
||||
def next_job(self):
|
||||
return self.jobs.pop(0) if self.jobs else None
|
||||
|
||||
def complete_job(self, job_id, result_summary):
|
||||
self.completed.append((job_id, result_summary))
|
||||
|
||||
def fail_job(self, job_id, error):
|
||||
self.failed.append((job_id, error))
|
||||
|
||||
def classify_batch(self, items):
|
||||
return []
|
||||
|
||||
|
||||
class _ExternalStopFlag:
|
||||
"""Mirrors the Windows Service wrapper's stop object: no signal registration,
|
||||
just a plain flag flippable from another thread."""
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
|
||||
|
||||
def test_run_sends_heartbeat_but_never_polls_when_paused_centrally(monkeypatch, tmp_path):
|
||||
stopper = _ExternalStopFlag()
|
||||
fake_hub = _FakeHubForLoop()
|
||||
installation_id = "00000000-0000-4000-8000-000000000222"
|
||||
calls = {"identity": 0, "next_job": 0, "heartbeat": 0, "heartbeat_installation_id": None}
|
||||
|
||||
def identity():
|
||||
calls["identity"] += 1
|
||||
return {"agent_id": 2, "installation_id": installation_id}
|
||||
|
||||
def heartbeat_metadata(payload):
|
||||
calls["heartbeat"] += 1
|
||||
calls["heartbeat_installation_id"] = payload["installation_id"]
|
||||
stopper.stop = True
|
||||
return {"agent_id": 2, "processing_enabled": False} # paused centrally by the Hub
|
||||
|
||||
def forbidden_next_job():
|
||||
calls["next_job"] += 1
|
||||
raise AssertionError("processing is paused centrally; next_job must not be called")
|
||||
|
||||
fake_hub.identity = identity
|
||||
fake_hub.heartbeat_metadata = heartbeat_metadata
|
||||
fake_hub.next_job = forbidden_next_job
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service.time, "sleep", lambda _seconds: None)
|
||||
|
||||
config = _config(
|
||||
agent_id="2",
|
||||
poll_interval_seconds=0,
|
||||
state_db=str(tmp_path / "agent.db"),
|
||||
)
|
||||
env = {
|
||||
"AGENT_ID": "2",
|
||||
"FIREBIRD_DATABASE_PATH": AUTHORIZED,
|
||||
"FIREBIRD_WRITES": "0",
|
||||
"FISCAL_MODE": "DRY_RUN",
|
||||
}
|
||||
|
||||
service.run(config, env, stopper=stopper)
|
||||
|
||||
assert calls == {
|
||||
"identity": 1,
|
||||
"next_job": 0,
|
||||
"heartbeat": 1,
|
||||
"heartbeat_installation_id": installation_id,
|
||||
}
|
||||
assert service.AgentState(config.state_db).stored_installation_id() == installation_id
|
||||
|
||||
|
||||
def test_company_context_for_heartbeat_resolves_once_and_caches(monkeypatch):
|
||||
"""CRT was never sent at heartbeat: the Agent already proves it (company_context.resolve, via preflight_company_context) for coverage/RTC
|
||||
jobs, but that value never reached the Hub. This locks in the fix: resolved once, cached, never re-queried once solid."""
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_preflight(env):
|
||||
calls["n"] += 1
|
||||
return {"status": "PASS", "company_uf": "BA", "company_crt": "1", "company_code": "000001"}
|
||||
|
||||
monkeypatch.setattr("gti_fiscal_agent.company_context.preflight_company_context", fake_preflight)
|
||||
result, reason = service._company_context_for_heartbeat({}, None)
|
||||
assert result == {"status": "PASS", "company_uf": "BA", "company_crt": "1", "company_code": "000001"} and reason is None and calls["n"] == 1
|
||||
result2, reason2 = service._company_context_for_heartbeat({}, result) # already cached: never re-queries Firebird
|
||||
assert result2 is result and reason2 is None and calls["n"] == 1
|
||||
|
||||
|
||||
def test_company_context_for_heartbeat_retries_while_unresolved_never_guesses(monkeypatch):
|
||||
def fake_preflight(env):
|
||||
return {"status": "FAIL", "classification": "UNRESOLVED", "reasons": ["UF_SOURCE_UNKNOWN"]}
|
||||
|
||||
monkeypatch.setattr("gti_fiscal_agent.company_context.preflight_company_context", fake_preflight)
|
||||
result, reason = service._company_context_for_heartbeat({}, None)
|
||||
assert result is None and "UNRESOLVED" in reason # never a guessed uf/crt - explicitly unresolved, retried next tick
|
||||
|
||||
|
||||
def test_run_heartbeat_carries_the_auto_discovered_company_crt(monkeypatch, tmp_path):
|
||||
"""End-to-end: once company context resolves, the CRT rides the heartbeat payload the Hub already knows how to store
|
||||
(Installation.company_crt) - closing the transport gap without any Hub change."""
|
||||
stopper = _ExternalStopFlag()
|
||||
fake_hub = _FakeHubForLoop()
|
||||
installation_id = "00000000-0000-4000-8000-000000000333"
|
||||
sent = {}
|
||||
|
||||
def identity():
|
||||
return {"agent_id": 2, "installation_id": installation_id}
|
||||
|
||||
def heartbeat_metadata(payload):
|
||||
sent.update(payload)
|
||||
stopper.stop = True
|
||||
return {"agent_id": 2, "processing_enabled": False}
|
||||
|
||||
fake_hub.identity = identity
|
||||
fake_hub.heartbeat_metadata = heartbeat_metadata
|
||||
fake_hub.next_job = lambda: (_ for _ in ()).throw(AssertionError("paused centrally; next_job must not be called"))
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr("gti_fiscal_agent.company_context.preflight_company_context",
|
||||
lambda env: {"status": "PASS", "company_uf": "BA", "company_crt": "1", "company_code": "000001", "company_contribuinte_ipi": "N"})
|
||||
|
||||
config = _config(agent_id="2", poll_interval_seconds=0, state_db=str(tmp_path / "agent.db"))
|
||||
env = {"AGENT_ID": "2", "FIREBIRD_DATABASE_PATH": AUTHORIZED, "FIREBIRD_WRITES": "0", "FISCAL_MODE": "DRY_RUN"}
|
||||
|
||||
service.run(config, env, stopper=stopper)
|
||||
|
||||
assert sent["company_uf"] == "BA" and sent["company_crt"] == "1" and sent["company_contribuinte_ipi"] == "N"
|
||||
|
||||
|
||||
def test_run_honors_an_externally_supplied_stopper(monkeypatch):
|
||||
# This is exactly the pattern the Windows Service wrapper uses: it never
|
||||
# constructs GracefulStop() (which would crash off the main thread), it
|
||||
# passes its own stop flag and flips it from the SvcStop handler.
|
||||
config = _config(poll_interval_seconds=0.001)
|
||||
fake_hub = _FakeHubForLoop()
|
||||
fake_hub.jobs = [] # no jobs; loop would run forever without an external stop
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service, "AgentState", lambda path: _FakeState())
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [])
|
||||
|
||||
stopper = _ExternalStopFlag()
|
||||
calls = {"n": 0}
|
||||
real_sleep = service.time.sleep
|
||||
|
||||
def counting_sleep(seconds):
|
||||
calls["n"] += 1
|
||||
if calls["n"] >= 3:
|
||||
stopper.stop = True
|
||||
|
||||
monkeypatch.setattr(service.time, "sleep", counting_sleep)
|
||||
service.run(config, {"AGENT_ID": "2", "INSTALLATION_ID": "00000000-0000-4000-8000-000000000222"}, stopper=stopper) # no max_iterations — only the external flag can stop it
|
||||
assert stopper.stop is True
|
||||
assert calls["n"] >= 3
|
||||
|
||||
|
||||
def test_run_survives_hub_down_then_processes_one_job_and_stops(monkeypatch):
|
||||
config = _config(poll_interval_seconds=0.001)
|
||||
fake_hub = _FakeHubForLoop()
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service, "AgentState", lambda path: _FakeState())
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [])
|
||||
monkeypatch.setattr(service, "build_job_coverage", lambda *a, **k: None)
|
||||
monkeypatch.setattr(service.time, "sleep", lambda seconds: None)
|
||||
|
||||
# first heartbeat call raises -> backoff branch; second call onward succeeds.
|
||||
service.run(config, {"AGENT_ID": "2", "INSTALLATION_ID": "00000000-0000-4000-8000-000000000222"}, max_iterations=4)
|
||||
|
||||
assert fake_hub.heartbeat_calls >= 2
|
||||
assert len(fake_hub.completed) == 1
|
||||
assert fake_hub.completed[0][0] == 1
|
||||
assert fake_hub.completed[0][1]["products_processed"] == 0
|
||||
@@ -0,0 +1,7 @@
|
||||
from gti_fiscal_agent.runner import to_hub_product
|
||||
|
||||
|
||||
def test_reference_schema_uses_classificacao_fiscal_as_canonical_ncm():
|
||||
product = {"CODIGO": "000276", "PRODUTO": "ABACAXI M/ JARRA", "NCM": "99999999", "CLASSIFICACAO_FISCAL": "08043000"}
|
||||
|
||||
assert to_hub_product(product)["current_ncm"] == "08043000"
|
||||
@@ -0,0 +1,56 @@
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent.state import AgentState
|
||||
|
||||
|
||||
def test_outbox_keeps_failed_work_pending(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
assert state.enqueue("000276", "source", "v1", "0.1.0")
|
||||
state.retry("000276")
|
||||
assert state.pending()[0]["status"] == "PENDING_API"
|
||||
state.result_received("000276", "source", "v1", "0.1.0")
|
||||
assert state.enqueue("000276", "source", "v1", "0.1.0") is False
|
||||
|
||||
|
||||
def test_ipi_write_log_records_per_product_audit(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
state.log_ipi_write("000276", "fp1", "EXACT_NCM", "IPI_RESOLVED", "SUCCEEDED", None, 3.25, "rowcount=1", "CHANGE_REQUIRED", "2026-09-18T00:00:00Z")
|
||||
log = state.ipi_write_log_for("000276")
|
||||
assert len(log) == 1
|
||||
assert log[0]["write_status"] == "SUCCEEDED"
|
||||
assert log[0]["before_value"] is None
|
||||
assert log[0]["proposed_value"] == "3.25"
|
||||
|
||||
|
||||
def test_installation_identity_requires_an_existing_or_explicit_id(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
|
||||
assert state.stored_installation_id() is None
|
||||
with pytest.raises(ValueError, match="installation_id is required"):
|
||||
state.installation_id()
|
||||
|
||||
assert state.connection.execute("SELECT COUNT(*) FROM installation_identity").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_installation_identity_persists_and_reuses_explicit_id(tmp_path):
|
||||
db_path = tmp_path / "agent.db"
|
||||
state = AgentState(db_path)
|
||||
existing_id = "00000000-0000-4000-8000-000000000222"
|
||||
|
||||
assert state.installation_id(existing_id) == existing_id
|
||||
assert state.installation_id() == existing_id
|
||||
assert state.stored_installation_id() == existing_id
|
||||
|
||||
restarted_state = AgentState(db_path)
|
||||
assert restarted_state.installation_id() == existing_id
|
||||
|
||||
|
||||
def test_installation_identity_rejects_conflicting_or_malformed_ids(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
existing_id = "00000000-0000-4000-8000-000000000222"
|
||||
state.installation_id(existing_id)
|
||||
|
||||
with pytest.raises(ValueError, match="conflicts"):
|
||||
state.installation_id("00000000-0000-4000-8000-000000000223")
|
||||
with pytest.raises(ValueError, match="UUID"):
|
||||
state.installation_id("not-a-uuid")
|
||||
@@ -0,0 +1,174 @@
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent import unified_fiscal as uf
|
||||
|
||||
pytestmark = pytest.mark.real_unified_fiscal
|
||||
|
||||
SCHEMA_ROWS = ([("C000025", c) for c in ("CODIGO", "CCLASSTRIB", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "PISCOFINS", "INCIDENCIA_PISCOFINS")]
|
||||
+ [("RTC_CCLASSTRIB", c) for c in ("CCLASSTRIB", "CSTIBSCBS", "INICIO_VIGENCIA", "FIM_VIGENCIA")] + [("RTC_CST", "CSTIBSCBS")] + [("PARAMETROS", "USAR_REFORMA_TRIBUTARIA")])
|
||||
PRODUCT_COLS = ("CODIGO", "CCLASSTRIB", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "PISCOFINS", "INCIDENCIA_PISCOFINS")
|
||||
PRODUCTS = [("1", None, None, None, None, None, None, None, "N", "NENHUMA"), ("2", "200003", None, "04", None, None, "04", None, "S", "MONOFÁSICA"), ("3", None, "2", "2", "2", "2", "2", "2", "N", "NENHUMA")]
|
||||
CLASSES = [("000001", "000", date(2025, 5, 5), None), ("200003", "200", date(2025, 5, 5), None)]
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
"""READ ONLY fake: answers only SELECTs and records every statement so tests can prove nothing else ran."""
|
||||
def __init__(self, param=0, params=None, schema=SCHEMA_ROWS, products=PRODUCTS, classes=CLASSES, cst=(("000",), ("200",))):
|
||||
self.param, self.params, self.schema, self.products, self.classes, self.cst, self.statements, self._rows = param, params, schema, products, classes, cst, [], []
|
||||
|
||||
def execute(self, sql, *args):
|
||||
self.statements.append(sql)
|
||||
text = " ".join(sql.split()).upper()
|
||||
assert text.startswith("SELECT"), f"non-SELECT statement: {sql}"
|
||||
if "FROM RDB$RELATION_FIELDS" in text:
|
||||
self._rows = list(self.schema)
|
||||
elif "FROM PARAMETROS" in text:
|
||||
self._rows = [(v,) for v in (self.params if self.params is not None else [self.param])]
|
||||
elif "RDB$RELATION_CONSTRAINTS" in text:
|
||||
self._rows = [(1,)]
|
||||
elif "FROM RTC_CCLASSTRIB" in text:
|
||||
self._rows = list(self.classes)
|
||||
elif "FROM RTC_CST" in text:
|
||||
self._rows = list(self.cst)
|
||||
elif "FROM C000025" in text:
|
||||
self._rows = list(self.products)
|
||||
else:
|
||||
raise AssertionError(sql)
|
||||
|
||||
def fetchall(self):
|
||||
return self._rows
|
||||
|
||||
def fetchone(self):
|
||||
return self._rows[0] if self._rows else None
|
||||
|
||||
|
||||
class FakeConn:
|
||||
def __init__(self, **kw):
|
||||
self.cur = FakeCursor(**kw)
|
||||
|
||||
def cursor(self):
|
||||
return self.cur
|
||||
|
||||
|
||||
def kinds(found):
|
||||
return sorted({b["type"] for b in found["blockers"]})
|
||||
|
||||
|
||||
def test_rtc_off_in_the_erp_is_out_of_scope_and_never_blocks():
|
||||
conn = FakeConn(param=0)
|
||||
out = uf.assess_from_connection(conn, {"crt": "1"}, today=date(2026, 9, 24))
|
||||
assert out["context"]["rtc_enabled"] is False and out["context"]["rtc_supported"] is True and out["context"]["rtc_catalog_valid"] is True
|
||||
assert not [b for f in out["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert kinds(out["findings"]["1"]) == [] and kinds(out["findings"]["2"]) == [] and kinds(out["findings"]["3"]) == [] # Simples + explicit "no special treatment": not applicable, typed "2" is only a warning
|
||||
assert all(s.strip().upper().startswith("SELECT") for s in conn.cur.statements) # read-only: SELECT statements only
|
||||
|
||||
|
||||
def test_rtc_on_in_the_erp_does_not_make_simples_cclasstrib_applicable_before_effective_date():
|
||||
out = uf.assess_from_connection(FakeConn(param=1), {"crt": "1"}, today=date(2026, 9, 24))
|
||||
assert out["context"]["rtc_enabled"] is True
|
||||
assert out["context"]["cclasstrib_applicability"]["state"] == "NOT_APPLICABLE"
|
||||
assert not [b for f in out["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
|
||||
|
||||
def test_rtc_on_reports_simples_cclasstrib_as_not_applicable_and_keeps_other_fields():
|
||||
products = [("1", None, None, None, None, None, None, None, "S", "MONOFÁSICA")]
|
||||
conn = FakeConn(param=1, products=products)
|
||||
|
||||
out = uf.assess_from_connection(conn, {"crt": "1"}, today=date(2026, 9, 25))
|
||||
|
||||
assert out["context"]["cclasstrib_applicability"]["state"] == "NOT_APPLICABLE"
|
||||
assert out["context"]["cclasstrib_applicability"]["source"] == "C000004.CRT + RTC effective-date rules"
|
||||
assert kinds(out["findings"]["1"]) == ["COFINS_CST_MISSING", "PIS_CST_MISSING"]
|
||||
assert all(sql.strip().upper().startswith("SELECT") for sql in conn.cur.statements)
|
||||
|
||||
|
||||
def test_cclasstrib_applicability_change_changes_product_input_fingerprint():
|
||||
simples = uf.assess_from_connection(FakeConn(param=1), {"crt": "1"}, today=date(2026, 9, 25))
|
||||
normal = uf.assess_from_connection(FakeConn(param=1), {"crt": "3"}, today=date(2026, 9, 25))
|
||||
|
||||
assert simples["findings"]["1"]["input_fingerprint"] != normal["findings"]["1"]["input_fingerprint"]
|
||||
assert kinds(simples["findings"]["1"]) == []
|
||||
assert any(blocker["category"] == "RTC" for blocker in normal["findings"]["1"]["blockers"])
|
||||
|
||||
|
||||
def test_simulation_flag_never_touches_the_erp_parameter():
|
||||
conn = FakeConn(param=0)
|
||||
on = uf.assess_from_connection(conn, {"crt": "1"}, rtc_override=True)
|
||||
assert on["context"]["rtc_enabled"] is True and on["context"]["cclasstrib_applicability"]["state"] == "NOT_APPLICABLE"
|
||||
assert not [b for f in on["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert on["context"]["rtc_param"] == "0" and on["context"]["rtc_source"] == "SIMULATION" and not any("UPDATE" in s.upper() for s in conn.cur.statements)
|
||||
|
||||
|
||||
def test_old_schema_without_rtc_columns_reports_unsupported_and_still_evaluates_pis_cofins():
|
||||
old = [("C000025", c) for c in ("CODIGO", "COD_SAIDA_PIS", "PISCOFINS", "INCIDENCIA_PISCOFINS")]
|
||||
rows = [("1", None, "N", "NENHUMA"), ("2", "049", "N", "NENHUMA")]
|
||||
out = uf.assess_from_connection(FakeConn(schema=old, products=rows, classes=[], cst=()), {"crt": "3"})
|
||||
assert out["context"]["rtc_supported"] is False and out["context"]["rtc_enabled"] is False
|
||||
assert kinds(out["findings"]["1"]) == [] and kinds(out["findings"]["2"]) == ["PIS_CST_INVALID"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value,expected", [(None, False), (0, False), ("0", False), (1, True), ("1", True), ("S", True), ("N", False)])
|
||||
def test_parameter_values_are_read_fail_closed(value, expected):
|
||||
caps = uf.detect_rtc_capabilities({"C000025": {"CCLASSTRIB"}, "RTC_CCLASSTRIB": {"CCLASSTRIB", "CSTIBSCBS"}, "RTC_CST": {"CSTIBSCBS"}}, param_value=value)
|
||||
assert caps["RTC_ENABLED_BY_PARAM"] is expected
|
||||
|
||||
|
||||
def test_catalog_with_orphan_or_duplicate_is_invalid_and_flagged_only_when_rtc_is_on():
|
||||
bad = FakeConn(param=1, classes=CLASSES + [("200003", "200", date(2025, 5, 5), None)]) # duplicate class
|
||||
on = uf.assess_from_connection(bad, {"crt": "3"})
|
||||
assert on["context"]["rtc_catalog_valid"] is False and kinds(on["findings"]["2"]) == ["RTC_CATALOG_INVALID"]
|
||||
off = uf.assess_from_connection(FakeConn(param=0, classes=CLASSES + [("200003", "200", date(2025, 5, 5), None)]), {"crt": "1"})
|
||||
assert not [b for f in off["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
|
||||
|
||||
def test_input_fingerprint_changes_with_the_fiscal_values_and_the_rtc_switch():
|
||||
a = uf.assess_from_connection(FakeConn(param=0), {"crt": "1"})["findings"]["1"]["input_fingerprint"]
|
||||
b = uf.assess_from_connection(FakeConn(param=1), {"crt": "1"})["findings"]["1"]["input_fingerprint"]
|
||||
changed = [("1", None, None, "99", None, None, None, None, "N", "NENHUMA")] + PRODUCTS[1:]
|
||||
c = uf.assess_from_connection(FakeConn(param=0, products=changed), {"crt": "1"})["findings"]["1"]["input_fingerprint"]
|
||||
assert len({a, b, c}) == 3 and re.fullmatch(r"[0-9a-f]{64}", a)
|
||||
|
||||
|
||||
def test_rtc_state_is_read_from_this_database_now_with_provenance_and_timestamp():
|
||||
out = uf.assess_from_connection(FakeConn(param=1), {"crt": "1"})
|
||||
ctx = out["context"]
|
||||
assert ctx["rtc_enabled"] is True and ctx["erp_rtc_enabled"] is True and ctx["rtc_source"] == "PARAMETROS.USAR_REFORMA_TRIBUTARIA" and ctx["rtc_param"] == "1" and ctx["rtc_param_rows"] == 1
|
||||
assert ctx["rtc_param_conflict"] is False and ctx["rtc_observed_at"].endswith("+00:00")
|
||||
assert ctx["cclasstrib_applicability"]["state"] == "NOT_APPLICABLE" and ctx["cclasstrib_applicability"]["source"] == "C000004.CRT + RTC effective-date rules"
|
||||
assert ctx["legal_effective_context"]["informational_only"] is True and ctx["legal_effective_context"]["crt"] == "1"
|
||||
|
||||
|
||||
def test_toggle_between_syncs_is_followed_in_both_directions_without_any_cache():
|
||||
conn = FakeConn(param=0)
|
||||
first = uf.assess_from_connection(conn, {"crt": "1"})
|
||||
assert first["context"]["rtc_enabled"] is False and not [b for f in first["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
conn.cur.param = 1 # the operator switches the reform on in the ERP between two syncs
|
||||
second = uf.assess_from_connection(conn, {"crt": "1"})
|
||||
assert second["context"]["rtc_enabled"] is True and second["context"]["cclasstrib_applicability"]["state"] == "NOT_APPLICABLE"
|
||||
assert not [b for f in second["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert second["findings"]["1"]["input_fingerprint"] != first["findings"]["1"]["input_fingerprint"] # a new blocker fingerprint: the old (RTC off) state is not reused
|
||||
conn.cur.param = 0 # and off again
|
||||
third = uf.assess_from_connection(conn, {"crt": "1"})
|
||||
assert third["context"]["rtc_enabled"] is False and not [b for f in third["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert third["findings"]["1"]["input_fingerprint"] == first["findings"]["1"]["input_fingerprint"]
|
||||
|
||||
|
||||
def test_conflicting_or_repeated_parameter_rows_never_guess_on():
|
||||
same = uf.assess_from_connection(FakeConn(params=[1, 1]), {"crt": "1"})["context"]
|
||||
assert same["rtc_enabled"] is True and same["rtc_param_rows"] == 2 and same["rtc_param_conflict"] is False
|
||||
conflict = uf.assess_from_connection(FakeConn(params=[1, 0]), {"crt": "1"})
|
||||
assert conflict["context"]["rtc_enabled"] is False and conflict["context"]["rtc_param_conflict"] is True
|
||||
assert not [b for f in conflict["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert uf.assess_from_connection(FakeConn(params=[]), {"crt": "1"})["context"]["rtc_enabled"] is False
|
||||
|
||||
|
||||
def test_non_canonical_cst_warnings_are_reported_without_blocking():
|
||||
rows = [("1", None, None, "4", None, None, "4", None, "N", "NENHUMA"), ("2", None, None, None, None, None, None, None, "N", "NENHUMA")]
|
||||
out = uf.assess_from_connection(FakeConn(products=rows), {"crt": "3"})
|
||||
assert kinds(out["findings"]["1"]) == [] and kinds(out["findings"]["2"]) == []
|
||||
w = out["context"]["non_canonical_warnings"]
|
||||
assert w["products"] == 1 and w["by_type"] == {"CST_COFINS_NON_CANONICAL_FORMAT": 1, "CST_PIS_NON_CANONICAL_FORMAT": 1} and w["samples"][0]["product_code"] == "1"
|
||||
assert out["context"]["pis_cofins_applicability"] == {"OPTIONAL": 2}
|
||||
@@ -0,0 +1,65 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
SERVICE_WRAPPER = Path(__file__).parents[1] / "service" / "gti_agent_windows_service.py"
|
||||
|
||||
|
||||
def load_service_wrapper():
|
||||
spec = importlib.util.spec_from_file_location("gti_agent_windows_service_test", SERVICE_WRAPPER)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_main_enters_scm_dispatcher_when_started_without_command(monkeypatch):
|
||||
service = load_service_wrapper()
|
||||
calls = []
|
||||
|
||||
class FakeServiceManager:
|
||||
def Initialize(self):
|
||||
calls.append("initialize")
|
||||
|
||||
def PrepareToHostSingle(self, service_class):
|
||||
calls.append(("prepare", service_class))
|
||||
|
||||
def StartServiceCtrlDispatcher(self):
|
||||
calls.append("dispatcher")
|
||||
|
||||
def forbidden_handle_command_line(service_class):
|
||||
raise AssertionError("SCM start must not be parsed as a CLI command")
|
||||
|
||||
monkeypatch.setattr(service, "servicemanager", FakeServiceManager())
|
||||
monkeypatch.setattr(service, "win32serviceutil", SimpleNamespace(HandleCommandLine=forbidden_handle_command_line))
|
||||
monkeypatch.setattr(service.sys, "argv", [str(SERVICE_WRAPPER)])
|
||||
|
||||
service.main()
|
||||
|
||||
assert calls == ["initialize", ("prepare", service.GTIFiscalAgentService), "dispatcher"]
|
||||
|
||||
|
||||
def test_main_preserves_pywin32_cli_commands(monkeypatch):
|
||||
service = load_service_wrapper()
|
||||
called = []
|
||||
|
||||
class ForbiddenServiceManager:
|
||||
def __getattr__(self, name):
|
||||
raise AssertionError("CLI commands must not enter the SCM dispatcher")
|
||||
|
||||
monkeypatch.setattr(service, "servicemanager", ForbiddenServiceManager())
|
||||
monkeypatch.setattr(service, "win32serviceutil", SimpleNamespace(HandleCommandLine=lambda cls: called.append(cls)))
|
||||
monkeypatch.setattr(service.sys, "argv", [str(SERVICE_WRAPPER), "start"])
|
||||
|
||||
service.main()
|
||||
|
||||
assert called == [service.GTIFiscalAgentService]
|
||||
|
||||
|
||||
def test_a_fatal_error_in_the_agent_loop_stops_the_service_instead_of_leaving_a_dead_one_running():
|
||||
text = SERVICE_WRAPPER.read_text(encoding="utf-8")
|
||||
body = text.split("agent_service.run(config, os.environ, stopper=self.stop_flag)", 1)[1].split("def main", 1)[0]
|
||||
assert "except BaseException" in body and "logger" not in body.split("except BaseException")[0][-40:] # wrapped in try/except
|
||||
assert "LogErrorMsg" in body and "self.stop_flag.stop = True" in body and "win32event.SetEvent(self.stop_event)" in body
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Agent side of the zero-touch model: delta sync via fingerprint cache, read-only guarantee,
|
||||
blockers/operational summary in the job result, central pause, Firebird credential resolution."""
|
||||
import json
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from test_run_agent_service import AUTHORIZED, _FakeExecutor, _config, service
|
||||
|
||||
|
||||
class FakeHub:
|
||||
def __init__(self):
|
||||
self.classified = []
|
||||
|
||||
def classify_batch(self, products):
|
||||
self.classified.append([p["product_id"] for p in products])
|
||||
return [{"status": "EXACT_NCM" if p["current_ncm"] else "INVALID_NCM", "ipi_status": "IPI_RESOLVED", "classification": {}} for p in products]
|
||||
|
||||
|
||||
def product(code, ncm="03055100", name="X"):
|
||||
return {"CODIGO": code, "PRODUTO": name, "EAN_GTIN": None, "CLASSIFICACAO_FISCAL": ncm, "NCM": None, "CFOP": "5102", "CST": None, "ALIQUOTA": None, "CEST": None, "CSOSN": "102", "IPI": None}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env_setup(monkeypatch, tmp_path):
|
||||
hub = FakeHub()
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: hub)
|
||||
monkeypatch.setattr(service, "build_job_coverage", lambda env, client, products, decisions: {"total_products": len(products)})
|
||||
state = service.AgentState(tmp_path / "s.db")
|
||||
return hub, state
|
||||
|
||||
|
||||
def run_job(kind, products, state, monkeypatch, config=None):
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
return service.execute_job({"job_id": 1, "kind": kind, "agent_id": 2, "payload": {}}, config or _config(), _FakeExecutor(), state, {})
|
||||
|
||||
|
||||
def test_initial_sync_processes_everything_and_reports_operational_summary(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
products = [product("1"), product("2"), product("3", ncm="")]
|
||||
result, error = run_job("INITIAL_SYNC", products, state, monkeypatch)
|
||||
assert error is None
|
||||
assert result["products_total"] == 3 and result["products_changed"] == 3 and hub.classified == [["1", "2", "3"]]
|
||||
op = result["operational"]
|
||||
assert op["TOTAL_PRODUCTS"] == 3 and op["COMPLETE"] == 2 and op["SAFE_BLOCKED"] == 1
|
||||
assert op["COMPLETE"] + op["CORRECTED"] + op["SAFE_BLOCKED"] + op["FAILED_TECHNICAL"] == op["TOTAL_PRODUCTS"] # invariant closes
|
||||
assert [b["product_code"] for b in result["blocked_products"]] == ["3"] and result["resolved_products"] == ["1", "2"]
|
||||
|
||||
|
||||
def test_incremental_sync_reclassifies_only_new_and_changed_products(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
base = [product("1"), product("2"), product("3")]
|
||||
run_job("INITIAL_SYNC", base, state, monkeypatch)
|
||||
hub.classified.clear()
|
||||
|
||||
result, error = run_job("SYNC_INCREMENTAL", base, state, monkeypatch) # nothing changed
|
||||
assert error is None and result["products_changed"] == 0 and hub.classified == [[]] or hub.classified == []
|
||||
assert result["operational"]["TOTAL_PRODUCTS"] == 3 # coverage still complete, rebuilt from the cache
|
||||
|
||||
hub.classified.clear()
|
||||
changed = [product("1"), product("2", ncm="", name="broken now"), product("3"), product("4")] # 2 changed, 4 new
|
||||
result, _ = run_job("SYNC_INCREMENTAL", changed, state, monkeypatch)
|
||||
assert sorted(hub.classified[0]) == ["2", "4"] and result["products_changed"] == 2
|
||||
assert result["operational"]["TOTAL_PRODUCTS"] == 4 and result["operational"]["SAFE_BLOCKED"] == 1
|
||||
|
||||
|
||||
def test_removed_product_leaves_local_cache_but_is_not_reported_as_resolved(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
run_job("INITIAL_SYNC", [product("1"), product("2")], state, monkeypatch)
|
||||
result, _ = run_job("SYNC_INCREMENTAL", [product("1")], state, monkeypatch)
|
||||
assert state.cached_decision("2") is None and result["operational"]["TOTAL_PRODUCTS"] == 1
|
||||
assert result["resolved_products"] == [] # no history is auto-resolved/deleted for a product that merely disappeared
|
||||
|
||||
|
||||
def test_every_sync_kind_is_read_only(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
for kind in ("INITIAL_SYNC", "SYNC_INCREMENTAL", "REPROCESS_ALL"):
|
||||
result, error = run_job(kind, [product("1")], state, monkeypatch)
|
||||
assert error is None and result["writes_attempted"] == 0 and result["writes_succeeded"] == 0 and result["firebird_writes_enabled"] is False
|
||||
with pytest.raises(RuntimeError, match="writes are disabled"):
|
||||
service.open_write_connection({}, AUTHORIZED, firebird_writes=False) # no write-capable connection can even be opened
|
||||
|
||||
|
||||
def test_hub_offline_during_job_fails_the_job_cleanly_no_crash(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
def boom(products): raise ConnectionError("hub down")
|
||||
hub.classify_batch = boom
|
||||
result, error = run_job("SYNC_INCREMENTAL", [product("9")], state, monkeypatch)
|
||||
assert result is None and "ConnectionError" in error # reported as a job failure, the service loop keeps running
|
||||
|
||||
|
||||
def test_firebird_offline_during_job_fails_the_job_cleanly(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
def down(env, codigos=None): raise OSError("Firebird service not running")
|
||||
monkeypatch.setattr(service, "read_products", down)
|
||||
result, error = service.execute_job({"job_id": 1, "kind": "SYNC_INCREMENTAL", "agent_id": 2, "payload": {}}, _config(), _FakeExecutor(), state, {})
|
||||
assert result is None and "OSError" in error
|
||||
|
||||
|
||||
def test_firebird_credentials_come_from_the_dpapi_file_not_agent_env(monkeypatch, tmp_path):
|
||||
from gti_fiscal_agent import credential_store
|
||||
path = tmp_path / "firebird.bin"
|
||||
monkeypatch.setattr(credential_store, "read_credential", lambda p, env=None: json.dumps({"user": "SYSDBA", "password": "s3cret"}) if str(p) == str(path) else (_ for _ in ()).throw(FileNotFoundError()))
|
||||
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "GTI_FIREBIRD_CREDENTIAL_FILE": str(path)}
|
||||
service.Config(env)
|
||||
assert env["FB_USER"] == "SYSDBA" and env["FB_PASSWORD"] == "s3cret"
|
||||
explicit = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "FB_PASSWORD": "dev", "FB_USER": "u", "GTI_FIREBIRD_CREDENTIAL_FILE": str(path)}
|
||||
service.Config(explicit)
|
||||
assert explicit["FB_PASSWORD"] == "dev" # an explicit env value still wins (dev override)
|
||||
|
||||
|
||||
def test_central_resume_after_pause_makes_agent_poll_again(monkeypatch, tmp_path):
|
||||
class Stopper: stop = False
|
||||
stopper = Stopper()
|
||||
states = iter([False, True]) # paused, then resumed
|
||||
calls = {"next_job": 0}
|
||||
|
||||
class Hub:
|
||||
def identity(self): return {"agent_id": 2, "installation_id": "00000000-0000-4000-8000-000000000222"}
|
||||
def heartbeat_metadata(self, payload): return {"processing_enabled": next(states)}
|
||||
def next_job(self):
|
||||
calls["next_job"] += 1; stopper.stop = True; return None
|
||||
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: Hub())
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service.time, "sleep", lambda s: None)
|
||||
service.run(_config(agent_id="2", poll_interval_seconds=0, state_db=str(tmp_path / "a.db")), {"AGENT_ID": "2"}, stopper=stopper)
|
||||
assert calls["next_job"] == 1 # first cycle paused (no poll), second cycle resumed (polled)
|
||||
|
||||
|
||||
def test_heartbeats_keep_flowing_while_a_long_job_runs(monkeypatch, tmp_path):
|
||||
beats = []
|
||||
|
||||
class Hub:
|
||||
def identity(self): return {"agent_id": 2, "installation_id": "00000000-0000-4000-8000-000000000222"}
|
||||
def heartbeat_metadata(self, payload): beats.append(payload["status"]); return {"processing_enabled": True}
|
||||
def next_job(self): return {"job_id": 5, "kind": "INITIAL_SYNC", "agent_id": 2, "payload": {}}
|
||||
def complete_job(self, job_id, summary): stopper.stop = True
|
||||
def fail_job(self, job_id, error): stopper.stop = True
|
||||
|
||||
class Stopper: stop = False
|
||||
stopper = Stopper()
|
||||
monkeypatch.setattr(service, "BUSY_HEARTBEAT_SECONDS", 0.05)
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: Hub())
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service, "execute_job", lambda *a, **k: (time.sleep(0.4), ({"ok": 1}, None))[1])
|
||||
service.run(_config(agent_id="2", poll_interval_seconds=0, state_db=str(tmp_path / "a.db")), {"AGENT_ID": "2"}, stopper=stopper)
|
||||
assert beats.count("BUSY") >= 3 # heartbeats during the job, not only before it
|
||||
|
||||
|
||||
def _findings(**by_code):
|
||||
out = {}
|
||||
for code, kinds in by_code.items():
|
||||
out[code] = {"blockers": [{"type": t, "category": c, "reason": "R"} for t, c in kinds], "applicability": {"PIS_COFINS": "REQUIRED", "RTC": "NOT_APPLICABLE"}, "input_fingerprint": "fp" + code, "needs_review": False}
|
||||
return out
|
||||
|
||||
|
||||
def test_unified_fiscal_blockers_change_the_single_coverage_and_the_single_queue(env_setup, monkeypatch):
|
||||
from gti_fiscal_agent import unified_fiscal
|
||||
findings = _findings(**{"2": [("PIS_CST_INVALID", "PIS"), ("COFINS_CST_INVALID", "COFINS")]})
|
||||
monkeypatch.setattr(unified_fiscal, "assess_from_connection", lambda conn, company, **kw: {"context": {"rtc_enabled": False, "rtc_supported": True, "rtc_catalog_valid": True}, "findings": findings})
|
||||
hub, state = env_setup
|
||||
result, error = run_job("INITIAL_SYNC", [product("1"), product("2"), product("3", ncm="")], state, monkeypatch)
|
||||
assert error is None
|
||||
op = result["operational"]
|
||||
assert op["TOTAL_PRODUCTS"] == 3 and op["COMPLETE_PRODUCTS"] == 1 and op["SAFE_BLOCKED"] == 2 and op["BLOCKED_FISCAL_DATA"] == 1 # one number, one definition
|
||||
assert op["COMPLETE_PRODUCTS"] + op["SAFE_BLOCKED"] + op["FAILED_TECHNICAL"] + op["UNCLASSIFIED"] == 3
|
||||
queue = sorted((b["product_code"], b["field"], b["reason_code"]) for b in result["blocked_products"])
|
||||
assert queue == [("2", "COFINS", "COFINS_CST_INVALID"), ("2", "PIS", "PIS_CST_INVALID"), ("3", "NCM", "INVALID_NCM")] # ONE queue: NCM and fiscal blockers together
|
||||
assert result["resolved_products"] == ["1"] # a fiscal blocker keeps the product out of resolved
|
||||
assert result["fiscal_context"]["rtc_enabled"] is False and result["fiscal_context"]["evaluated_all"] is True
|
||||
assert "expanded_scope" not in result
|
||||
|
||||
|
||||
def test_rtc_context_is_reported_but_blockers_only_exist_when_the_evaluator_produced_them(env_setup, monkeypatch):
|
||||
from gti_fiscal_agent import unified_fiscal
|
||||
monkeypatch.setattr(unified_fiscal, "assess_from_connection", lambda conn, company, **kw: {"context": {"rtc_enabled": True, "rtc_supported": True, "rtc_catalog_valid": True}, "findings": _findings(**{"1": [("CCLASSTRIB_MISSING", "RTC")]})})
|
||||
hub, state = env_setup
|
||||
result, _ = run_job("REPROCESS_ALL", [product("1"), product("2")], state, monkeypatch)
|
||||
assert [b["reason_code"] for b in result["blocked_products"]] == ["CCLASSTRIB_MISSING"] and result["blocked_products"][0]["field"] == "RTC"
|
||||
assert result["fiscal_context"]["rtc_enabled"] is True and result["operational"]["BLOCKED_FISCAL_DATA"] == 1
|
||||
|
||||
|
||||
def test_partial_reprocess_does_not_claim_a_full_fiscal_evaluation(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
result, _ = run_job("REPROCESS_PRODUCTS", [product("1")], state, monkeypatch)
|
||||
assert result["fiscal_context"] is None and result["operational"] is None
|
||||
|
||||
|
||||
def test_rtc_toggle_between_two_syncs_is_followed_by_the_next_job(env_setup, monkeypatch):
|
||||
"""The Agent asks the database EVERY sync: 0 -> 1 makes the next job report RTC blockers, 1 -> 0 makes the following job report none (Hub then closes the old ones)."""
|
||||
from gti_fiscal_agent import unified_fiscal
|
||||
switch = {"on": False}
|
||||
|
||||
def assess(conn, company, **kw):
|
||||
found = _findings(**{"1": [("CCLASSTRIB_MISSING", "RTC")]}) if switch["on"] else {}
|
||||
return {"context": {"rtc_enabled": switch["on"], "rtc_supported": True, "rtc_catalog_valid": True, "rtc_source": "PARAMETROS.USAR_REFORMA_TRIBUTARIA", "rtc_observed_at": "2026-09-24T20:00:00+00:00"}, "findings": found}
|
||||
monkeypatch.setattr(unified_fiscal, "assess_from_connection", assess)
|
||||
hub, state = env_setup
|
||||
products = [product("1"), product("2")]
|
||||
off1, _ = run_job("INITIAL_SYNC", products, state, monkeypatch)
|
||||
switch["on"] = True
|
||||
on, _ = run_job("SYNC_INCREMENTAL", products, state, monkeypatch) # nothing changed in C000025: the delta cache must not hide the RTC change
|
||||
switch["on"] = False
|
||||
off2, _ = run_job("SYNC_INCREMENTAL", products, state, monkeypatch)
|
||||
assert off1["blocked_products"] == [] and off1["fiscal_context"]["rtc_enabled"] is False
|
||||
assert [b["field"] for b in on["blocked_products"]] == ["RTC"] and on["fiscal_context"]["rtc_enabled"] is True and on["operational"]["BLOCKED_FISCAL_DATA"] == 1
|
||||
assert on["fiscal_context"]["rtc_observed_at"] and on["fiscal_context"]["rtc_source"] == "PARAMETROS.USAR_REFORMA_TRIBUTARIA"
|
||||
assert off2["blocked_products"] == [] and off2["fiscal_context"]["rtc_enabled"] is False and off2["fiscal_context"]["evaluated_all"] is True
|
||||
assert off2["operational"]["COMPLETE_PRODUCTS"] == 2
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Entry point: python audit_firebird_database.py <FDB> (read-only; see scripts/audit_firebird_database.py)."""
|
||||
import runpy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.argv[0] = str(Path(__file__).with_name("scripts") / "audit_firebird_database.py")
|
||||
runpy.run_path(sys.argv[0], run_name="__main__")
|
||||
@@ -0,0 +1,7 @@
|
||||
FROM python:3.12-slim
|
||||
WORKDIR /app
|
||||
COPY central/pyproject.toml /tmp/central/pyproject.toml
|
||||
RUN pip install --no-cache-dir /tmp/central
|
||||
COPY central /app/central
|
||||
ENV PYTHONPATH=/app
|
||||
CMD ["sh", "-c", "alembic -c central/alembic.ini upgrade head && uvicorn central.app.main:app --host 0.0.0.0 --port 8000"]
|
||||
@@ -0,0 +1 @@
|
||||
"""GTI Fiscal Hub package."""
|
||||
@@ -0,0 +1,3 @@
|
||||
[alembic]
|
||||
script_location = central/alembic
|
||||
sqlalchemy.url = postgresql+psycopg://gti@postgres:5432/gti_fiscal
|
||||
@@ -0,0 +1,21 @@
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
from central.app.models import Base
|
||||
from central.app.config import settings
|
||||
config = context.config
|
||||
target_metadata = Base.metadata
|
||||
def run_migrations_offline():
|
||||
context.configure(url=settings.database_url, target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"})
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
def run_migrations_online():
|
||||
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||
engine = engine_from_config(config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool)
|
||||
with engine.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,99 @@
|
||||
"""initial hub persistence
|
||||
|
||||
Revision ID: 0001
|
||||
|
||||
Frozen, explicit baseline. This revision used to run
|
||||
``Base.metadata.create_all()``, which builds the schema from the *current*
|
||||
models: on an empty database it created tables/columns that 0002..0018 then
|
||||
tried to create again (first collision: ``datasets.raw_sha256`` in 0002), so
|
||||
``alembic upgrade head`` could never run from zero.
|
||||
|
||||
This body reproduces exactly what 0001 created on the databases that went
|
||||
through the real history: the seven base tables, *before* 0002 added
|
||||
``datasets.raw_sha256`` and 0006 added the agent status/timestamp columns.
|
||||
Constraint and index names match the ones those databases already have
|
||||
(PostgreSQL defaults for PK/FK/UNIQUE, ``ix_<table>_<column>`` for indexes).
|
||||
|
||||
Databases already at 0001 or later never re-run this revision, so changing
|
||||
its body only affects new installations. It must never import the ORM
|
||||
models again: a migration describes one point in history, not the present.
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0001"
|
||||
down_revision = None
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"tenants",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("name", sa.String(128), nullable=False, unique=True),
|
||||
)
|
||||
op.create_table(
|
||||
"datasets",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("kind", sa.String(32), nullable=False),
|
||||
sa.Column("provider", sa.String(128), nullable=False),
|
||||
sa.Column("sha256", sa.String(64), nullable=False),
|
||||
sa.Column("version_label", sa.String(256), nullable=False),
|
||||
sa.Column("act", sa.Text(), nullable=False),
|
||||
sa.Column("source_url", sa.Text(), nullable=False),
|
||||
sa.Column("downloaded_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("record_count", sa.Integer(), nullable=False),
|
||||
sa.UniqueConstraint("provider", "sha256", name="uq_dataset_provider_hash"),
|
||||
)
|
||||
op.create_index("ix_datasets_kind", "datasets", ["kind"])
|
||||
op.create_index("ix_datasets_status", "datasets", ["status"])
|
||||
op.create_table(
|
||||
"ncm_entries",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("dataset_id", sa.Integer(), sa.ForeignKey("datasets.id"), nullable=False),
|
||||
sa.Column("code", sa.String(16), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=False),
|
||||
sa.Column("valid_from", sa.String(32), nullable=False),
|
||||
sa.Column("valid_to", sa.String(32), nullable=False),
|
||||
sa.UniqueConstraint("dataset_id", "code", "valid_from", "valid_to", name="uq_ncm_versioned_entry"),
|
||||
)
|
||||
op.create_index("ix_ncm_entries_dataset_id", "ncm_entries", ["dataset_id"])
|
||||
op.create_index("ix_ncm_entries_code", "ncm_entries", ["code"])
|
||||
op.create_table(
|
||||
"agents",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False),
|
||||
sa.Column("name", sa.String(128), nullable=False),
|
||||
sa.Column("token_hash", sa.String(64), nullable=False, unique=True),
|
||||
)
|
||||
op.create_index("ix_agents_tenant_id", "agents", ["tenant_id"])
|
||||
op.create_table(
|
||||
"classification_cache",
|
||||
sa.Column("fingerprint", sa.String(64), primary_key=True),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False),
|
||||
sa.Column("result", sa.JSON(), nullable=False),
|
||||
)
|
||||
op.create_index("ix_classification_cache_tenant_id", "classification_cache", ["tenant_id"])
|
||||
op.create_table(
|
||||
"classification_requests",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False),
|
||||
sa.Column("agent_id", sa.Integer(), sa.ForeignKey("agents.id"), nullable=False),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("payload", sa.JSON(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
)
|
||||
op.create_index("ix_classification_requests_tenant_id", "classification_requests", ["tenant_id"])
|
||||
op.create_index("ix_classification_requests_agent_id", "classification_requests", ["agent_id"])
|
||||
op.create_table(
|
||||
"classification_results",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("request_id", sa.Integer(), sa.ForeignKey("classification_requests.id"), nullable=False, unique=True),
|
||||
sa.Column("result", sa.JSON(), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
for table in ("classification_results", "classification_requests", "classification_cache", "agents", "ncm_entries", "datasets", "tenants"):
|
||||
op.drop_table(table)
|
||||
@@ -0,0 +1,31 @@
|
||||
"""add CEST links and official legacy NCM evidence
|
||||
|
||||
Revision ID: 0002
|
||||
Revises: 0001
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
|
||||
|
||||
revision = "0002"
|
||||
down_revision = "0001"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("datasets", Column("raw_sha256", String(64), nullable=True))
|
||||
op.create_table("legacy_ncm_entries", Column("code", String(16), primary_key=True), Column("source", Text, nullable=False), Column("source_date", String(32), nullable=False), Column("reason", Text, nullable=False), Column("successor_code", String(16), nullable=True))
|
||||
op.create_table("cest_entries", Column("id", Integer, primary_key=True), Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False), Column("code", String(16), nullable=False), Column("description", Text, nullable=False), Column("annex", String(8), nullable=False), UniqueConstraint("dataset_id", "code", name="uq_cest_versioned_entry"))
|
||||
op.create_table("cest_ncm_entries", Column("id", Integer, primary_key=True), Column("cest_entry_id", Integer, ForeignKey("cest_entries.id"), nullable=False), Column("ncm_code", String(16), nullable=False), UniqueConstraint("cest_entry_id", "ncm_code", name="uq_cest_ncm_link"))
|
||||
op.create_index("ix_cest_entries_dataset_id", "cest_entries", ["dataset_id"])
|
||||
op.create_index("ix_cest_entries_code", "cest_entries", ["code"])
|
||||
op.create_index("ix_cest_ncm_entries_cest_entry_id", "cest_ncm_entries", ["cest_entry_id"])
|
||||
op.create_index("ix_cest_ncm_entries_ncm_code", "cest_ncm_entries", ["ncm_code"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("cest_ncm_entries")
|
||||
op.drop_table("cest_entries")
|
||||
op.drop_table("legacy_ncm_entries")
|
||||
op.drop_column("datasets", "raw_sha256")
|
||||
@@ -0,0 +1,20 @@
|
||||
"""add normalized TIPI entries
|
||||
|
||||
Revision ID: 0003
|
||||
Revises: 0002
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, Float, ForeignKey, Integer, JSON, String, Text
|
||||
|
||||
revision = "0003"
|
||||
down_revision = "0002"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
def upgrade():
|
||||
op.create_table("tipi_entries", Column("id", Integer, primary_key=True), Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False), Column("ncm", String(8), nullable=False), Column("ex", String(32)), Column("description", Text, nullable=False), Column("ipi_rate", Float, nullable=True), Column("ipi_rate_text", String(32)), Column("source_row", Integer, nullable=False), Column("metadata_json", JSON, nullable=False))
|
||||
op.create_index("ix_tipi_entries_dataset_id", "tipi_entries", ["dataset_id"])
|
||||
op.create_index("ix_tipi_entries_ncm", "tipi_entries", ["ncm"])
|
||||
op.create_index("ix_tipi_entries_ncm_ex", "tipi_entries", ["ncm", "ex"])
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("tipi_entries")
|
||||
@@ -0,0 +1,16 @@
|
||||
"""add BA ST MVA entries
|
||||
|
||||
Revision ID: 0004
|
||||
Revises: 0003
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, ForeignKey, Integer, JSON, String, Text
|
||||
revision = "0004"
|
||||
down_revision = "0003"
|
||||
branch_labels = depends_on = None
|
||||
def upgrade():
|
||||
op.create_table("ba_st_mva_entries", Column("id", Integer, primary_key=True), Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False), Column("item", String(32), nullable=False), Column("segment", Text), Column("cest", String(16)), Column("ncm_sh", String(32)), Column("description", Text), Column("mva_original", String(32)), Column("mva_adjusted", String(32)), Column("agreement_info", Text), Column("source_page", Integer, nullable=False), Column("metadata_json", JSON, nullable=False))
|
||||
op.create_index("ix_ba_st_mva_entries_dataset_id", "ba_st_mva_entries", ["dataset_id"])
|
||||
op.create_index("ix_ba_st_mva_entries_cest", "ba_st_mva_entries", ["cest"])
|
||||
op.create_index("ix_ba_st_mva_entries_ncm_sh", "ba_st_mva_entries", ["ncm_sh"])
|
||||
def downgrade(): op.drop_table("ba_st_mva_entries")
|
||||
@@ -0,0 +1,15 @@
|
||||
"""normalize BA ST NCM SH
|
||||
|
||||
Revision ID: 0005
|
||||
Revises: 0004
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, ForeignKey, Integer, JSON, String, UniqueConstraint
|
||||
revision="0005"; down_revision="0004"; branch_labels=depends_on=None
|
||||
def upgrade():
|
||||
op.drop_index("ix_ba_st_mva_entries_ncm_sh", table_name="ba_st_mva_entries")
|
||||
op.drop_column("ba_st_mva_entries", "ncm_sh")
|
||||
op.create_table("ba_st_mva_ncm_sh", Column("id", Integer, primary_key=True), Column("entry_id", Integer, ForeignKey("ba_st_mva_entries.id"), nullable=False), Column("ncm_sh", String(32), nullable=False), Column("metadata_json", JSON, nullable=False), UniqueConstraint("entry_id", "ncm_sh", name="uq_ba_st_mva_entry_ncm_sh"))
|
||||
op.create_index("ix_ba_st_mva_ncm_sh_entry_id", "ba_st_mva_ncm_sh", ["entry_id"])
|
||||
op.create_index("ix_ba_st_mva_ncm_sh_ncm_sh", "ba_st_mva_ncm_sh", ["ncm_sh"])
|
||||
def downgrade(): pass
|
||||
@@ -0,0 +1,43 @@
|
||||
"""agent status fields and jobs table
|
||||
|
||||
Revision ID: 0006
|
||||
Revises: 0005
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, JSON, String, Text
|
||||
|
||||
revision = "0006"; down_revision = "0005"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("agents", Column("enabled", Boolean, server_default="true", nullable=False))
|
||||
op.add_column("agents", Column("last_seen_at", DateTime(timezone=True), nullable=True))
|
||||
op.add_column("agents", Column("last_status", String(32), nullable=True))
|
||||
op.add_column("agents", Column("schema_fingerprint", String(64), nullable=True))
|
||||
op.add_column("agents", Column("agent_version", String(32), nullable=True))
|
||||
op.add_column("agents", Column("created_at", DateTime(timezone=True), server_default="now()", nullable=False))
|
||||
op.add_column("agents", Column("updated_at", DateTime(timezone=True), server_default="now()", nullable=False))
|
||||
|
||||
op.create_table(
|
||||
"jobs",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("agent_id", Integer, ForeignKey("agents.id"), nullable=False),
|
||||
Column("kind", String(32), nullable=False),
|
||||
Column("status", String(16), nullable=False, server_default="PENDING"),
|
||||
Column("payload", JSON, nullable=False),
|
||||
Column("created_at", DateTime(timezone=True), server_default="now()", nullable=False),
|
||||
Column("started_at", DateTime(timezone=True), nullable=True),
|
||||
Column("finished_at", DateTime(timezone=True), nullable=True),
|
||||
Column("attempts", Integer, nullable=False, server_default="0"),
|
||||
Column("result_summary", JSON, nullable=True),
|
||||
Column("error", Text, nullable=True),
|
||||
)
|
||||
op.create_index("ix_jobs_agent_id", "jobs", ["agent_id"])
|
||||
op.create_index("ix_jobs_kind", "jobs", ["kind"])
|
||||
op.create_index("ix_jobs_status", "jobs", ["status"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("jobs")
|
||||
for column in ("enabled", "last_seen_at", "last_status", "schema_fingerprint", "agent_version", "created_at", "updated_at"):
|
||||
op.drop_column("agents", column)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""fiscal_code_entries table for ORIGEM/CST_ICMS/CSOSN/CST_PIS/CST_COFINS
|
||||
|
||||
Revision ID: 0007
|
||||
Revises: 0006
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, ForeignKey, Integer, JSON, String, Text, UniqueConstraint
|
||||
|
||||
revision = "0007"; down_revision = "0006"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"fiscal_code_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("domain", String(32), nullable=False),
|
||||
Column("code", String(8), nullable=False),
|
||||
Column("description", Text, nullable=False),
|
||||
Column("source_section", String(96), nullable=False),
|
||||
Column("metadata_json", JSON, nullable=False),
|
||||
UniqueConstraint("dataset_id", "domain", "code", name="uq_fiscal_code_entry"),
|
||||
)
|
||||
op.create_index("ix_fiscal_code_entries_dataset_id", "fiscal_code_entries", ["dataset_id"])
|
||||
op.create_index("ix_fiscal_code_entries_domain", "fiscal_code_entries", ["domain"])
|
||||
op.create_index("ix_fiscal_code_entries_code", "fiscal_code_entries", ["code"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("fiscal_code_entries")
|
||||
@@ -0,0 +1,124 @@
|
||||
"""CFOP/FCP/ANP/NCM_UTRIB/RTC_BIOFUEL_INDEX/RTC_CCREDPRES tables
|
||||
|
||||
Revision ID: 0008
|
||||
Revises: 0007
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Boolean, Column, Float, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
|
||||
revision = "0008"; down_revision = "0007"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"cfop_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("code", String(8), nullable=False),
|
||||
Column("title", Text, nullable=False),
|
||||
Column("description", Text, nullable=True),
|
||||
Column("effective_from", String(32), nullable=True),
|
||||
Column("effective_to", String(32), nullable=True),
|
||||
Column("ind_nfe", String(4), nullable=True),
|
||||
Column("ind_comunica", String(4), nullable=True),
|
||||
Column("ind_transp", String(4), nullable=True),
|
||||
Column("ind_devol", String(4), nullable=True),
|
||||
Column("ind_retor", String(4), nullable=True),
|
||||
Column("ind_anula", String(4), nullable=True),
|
||||
Column("ind_remes", String(4), nullable=True),
|
||||
Column("ind_comb", String(4), nullable=True),
|
||||
Column("ind_exc_ibscbs", String(4), nullable=True),
|
||||
UniqueConstraint("dataset_id", "code", name="uq_cfop_entry"),
|
||||
)
|
||||
op.create_index("ix_cfop_entries_dataset_id", "cfop_entries", ["dataset_id"])
|
||||
op.create_index("ix_cfop_entries_code", "cfop_entries", ["code"])
|
||||
|
||||
op.create_table(
|
||||
"fcp_rates",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("uf_code", String(4), nullable=False),
|
||||
Column("uf", String(4), nullable=False),
|
||||
Column("uf_name", String(64), nullable=False),
|
||||
Column("aliq_1", String(32), nullable=True),
|
||||
Column("aliq_2", String(32), nullable=True),
|
||||
Column("aliq_3", String(32), nullable=True),
|
||||
Column("observacao", Text, nullable=True),
|
||||
UniqueConstraint("dataset_id", "uf", name="uq_fcp_rate_uf"),
|
||||
)
|
||||
op.create_index("ix_fcp_rates_dataset_id", "fcp_rates", ["dataset_id"])
|
||||
op.create_index("ix_fcp_rates_uf", "fcp_rates", ["uf"])
|
||||
|
||||
op.create_table(
|
||||
"anp_product_codes",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("cprod_anp", String(16), nullable=False),
|
||||
UniqueConstraint("dataset_id", "cprod_anp", name="uq_anp_product_code"),
|
||||
)
|
||||
op.create_index("ix_anp_product_codes_dataset_id", "anp_product_codes", ["dataset_id"])
|
||||
op.create_index("ix_anp_product_codes_cprod_anp", "anp_product_codes", ["cprod_anp"])
|
||||
|
||||
op.create_table(
|
||||
"ncm_utrib_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("ncm", String(16), nullable=False),
|
||||
Column("effective_from", String(32), nullable=True),
|
||||
Column("effective_to", String(32), nullable=True),
|
||||
Column("utrib_abbreviation", String(16), nullable=True),
|
||||
Column("utrib_description", Text, nullable=True),
|
||||
UniqueConstraint("dataset_id", "ncm", name="uq_ncm_utrib_entry"),
|
||||
)
|
||||
op.create_index("ix_ncm_utrib_entries_dataset_id", "ncm_utrib_entries", ["dataset_id"])
|
||||
op.create_index("ix_ncm_utrib_entries_ncm", "ncm_utrib_entries", ["ncm"])
|
||||
|
||||
op.create_table(
|
||||
"rtc_biofuel_index_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("cprod_anp", String(16), nullable=True),
|
||||
Column("description", Text, nullable=True),
|
||||
Column("p_bio_percent", Float, nullable=True),
|
||||
Column("effective_from", String(32), nullable=True),
|
||||
Column("effective_to", String(32), nullable=True),
|
||||
Column("is_historical", Boolean, nullable=False, server_default="false"),
|
||||
)
|
||||
op.create_index("ix_rtc_biofuel_index_entries_dataset_id", "rtc_biofuel_index_entries", ["dataset_id"])
|
||||
op.create_index("ix_rtc_biofuel_index_entries_cprod_anp", "rtc_biofuel_index_entries", ["cprod_anp"])
|
||||
|
||||
op.create_table(
|
||||
"rtc_cred_pres_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("code", String(8), nullable=False),
|
||||
Column("description", Text, nullable=False),
|
||||
Column("legal_basis", Text, nullable=True),
|
||||
Column("apropria_via_nf", Boolean, nullable=True),
|
||||
Column("apropria_via_evento", Boolean, nullable=True),
|
||||
Column("ind_deduz_cred_pres", String(8), nullable=True),
|
||||
Column("ind_g_cbs_cred_pres", String(8), nullable=True),
|
||||
Column("ind_g_ibs_cred_pres", String(8), nullable=True),
|
||||
Column("aliquota_cbs_text", Text, nullable=True),
|
||||
Column("aliquota_ibs_text", Text, nullable=True),
|
||||
Column("p_aliq_cred_pres_cbs", Text, nullable=True),
|
||||
Column("p_aliq_cred_pres_ibs", Text, nullable=True),
|
||||
Column("p_red_transicao_ibs", Text, nullable=True),
|
||||
Column("cclass_nota_referenciada", String(16), nullable=True),
|
||||
Column("d_ini_vig_cbs", String(32), nullable=True),
|
||||
Column("d_fim_vig_cbs", String(32), nullable=True),
|
||||
Column("d_ini_vig_ibs", String(32), nullable=True),
|
||||
Column("d_fim_vig_ibs", String(32), nullable=True),
|
||||
UniqueConstraint("dataset_id", "code", name="uq_rtc_cred_pres_code"),
|
||||
)
|
||||
op.create_index("ix_rtc_cred_pres_entries_dataset_id", "rtc_cred_pres_entries", ["dataset_id"])
|
||||
op.create_index("ix_rtc_cred_pres_entries_code", "rtc_cred_pres_entries", ["code"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("rtc_cred_pres_entries")
|
||||
op.drop_table("rtc_biofuel_index_entries")
|
||||
op.drop_table("ncm_utrib_entries")
|
||||
op.drop_table("anp_product_codes")
|
||||
op.drop_table("fcp_rates")
|
||||
op.drop_table("cfop_entries")
|
||||
@@ -0,0 +1,20 @@
|
||||
"""widen rtc_cred_pres_entries.cclass_nota_referenciada to Text
|
||||
|
||||
Some official rows carry free text ("cClass de alíquota zero") in this
|
||||
column, not just the numeric code — VARCHAR(16) truncated them.
|
||||
|
||||
Revision ID: 0009
|
||||
Revises: 0008
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Text
|
||||
|
||||
revision = "0009"; down_revision = "0008"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.alter_column("rtc_cred_pres_entries", "cclass_nota_referenciada", type_=Text, existing_nullable=True)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.alter_column("rtc_cred_pres_entries", "cclass_nota_referenciada", type_=None, existing_nullable=True)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""RTC cClassTrib, CST-IBS/CBS and CBS rate tables
|
||||
|
||||
Revision ID: 0010
|
||||
Revises: 0009
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import JSON, Column, Float, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
|
||||
revision = "0010"; down_revision = "0009"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def _fk():
|
||||
return Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False, index=True)
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"rtc_cst_ibscbs_entries",
|
||||
Column("id", Integer, primary_key=True), _fk(),
|
||||
Column("cst", String(8), nullable=False, index=True),
|
||||
Column("description", Text, nullable=False),
|
||||
Column("indicators", JSON, nullable=False),
|
||||
UniqueConstraint("dataset_id", "cst", name="uq_rtc_cst_ibscbs"),
|
||||
)
|
||||
op.create_table(
|
||||
"rtc_cclasstrib_entries",
|
||||
Column("id", Integer, primary_key=True), _fk(),
|
||||
Column("cst", String(8), nullable=False, index=True),
|
||||
Column("cst_description", Text), Column("cclasstrib", String(8), nullable=False, index=True),
|
||||
Column("name", Text), Column("description", Text), Column("lc_redacao", Text), Column("lc_214", Text),
|
||||
Column("regulamento_cbs", Text), Column("regulamento_ibs", Text),
|
||||
Column("tipo_aliquota", String(64)), Column("p_red_ibs", String(32)), Column("p_red_cbs", String(32)),
|
||||
Column("d_ini_vig", String(32)), Column("d_fim_vig", String(32)), Column("data_atualizacao", String(32)),
|
||||
Column("anexo", Text), Column("link", Text), Column("indicators", JSON, nullable=False),
|
||||
UniqueConstraint("dataset_id", "cclasstrib", name="uq_rtc_cclasstrib"),
|
||||
)
|
||||
op.create_table(
|
||||
"rtc_cbs_rate_entries",
|
||||
Column("id", Integer, primary_key=True), _fk(),
|
||||
Column("year", Integer, nullable=False, index=True),
|
||||
Column("rate_text", Text, nullable=False), Column("rate", Float),
|
||||
UniqueConstraint("dataset_id", "year", name="uq_rtc_cbs_rate_year"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
for t in ("rtc_cbs_rate_entries", "rtc_cclasstrib_entries", "rtc_cst_ibscbs_entries"):
|
||||
op.drop_table(t)
|
||||
@@ -0,0 +1,31 @@
|
||||
"""operator fiscal context entries (facts the ERP does not know: role, origin, ST retention, treatment, ...)
|
||||
|
||||
Revision ID: 0011
|
||||
Revises: 0010
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
|
||||
revision = "0011"; down_revision = "0010"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"fiscal_context_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("tenant_id", Integer, ForeignKey("tenants.id"), nullable=False, index=True),
|
||||
Column("company_key", String(64), nullable=False, index=True),
|
||||
Column("scope", String(8), nullable=False),
|
||||
Column("scope_key", String(64), nullable=False, server_default=""),
|
||||
Column("key", String(48), nullable=False),
|
||||
Column("value", Text, nullable=False),
|
||||
Column("source", String(16), nullable=False, server_default="OPERATOR"),
|
||||
Column("reason", Text, nullable=True),
|
||||
Column("created_at", DateTime(timezone=True), server_default=func.now()),
|
||||
Column("updated_at", DateTime(timezone=True), server_default=func.now()),
|
||||
UniqueConstraint("tenant_id", "company_key", "scope", "scope_key", "key", name="uq_fiscal_context_entry"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("fiscal_context_entries")
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Migration 0012: AI semantic layer V1 (KB, tenant context, work items, runs).
|
||||
|
||||
Revision ID: 0012
|
||||
Revises: 0011
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, DateTime, Float, ForeignKey, Integer, JSON, String, Text, UniqueConstraint, func
|
||||
|
||||
revision = "0012"
|
||||
down_revision = "0011"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table("kb_products", Column("id", Integer, primary_key=True), Column("fingerprint", String(64), nullable=False, unique=True), Column("gtin", String(16), unique=True), Column("identity_kind", String(16), nullable=False, server_default="FINGERPRINT"), Column("normalized_description", Text, nullable=False, server_default=""), Column("brand", Text), Column("unit", String(32)), Column("package_quantity", String(32)), Column("category", Text), Column("semantic_attributes", JSON, nullable=False), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("updated_at", DateTime(timezone=True), server_default=func.now()))
|
||||
op.create_index("ix_kb_products_gtin", "kb_products", ["gtin"])
|
||||
op.create_table("kb_knowledge", Column("id", Integer, primary_key=True), Column("kb_product_id", Integer, ForeignKey("kb_products.id"), nullable=False), Column("knowledge_key", String(48), nullable=False), Column("value", Text, nullable=False), Column("source", String(32), nullable=False), Column("confidence", Float, nullable=False, server_default="0"), Column("evidence", JSON, nullable=False), Column("model", String(64)), Column("prompt_version", String(32)), Column("dataset_versions", JSON, nullable=False), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("validated_at", DateTime(timezone=True)), UniqueConstraint("kb_product_id", "knowledge_key", "value", name="uq_kb_knowledge_value"))
|
||||
op.create_index("ix_kb_knowledge_kb_product_id", "kb_knowledge", ["kb_product_id"])
|
||||
op.create_index("ix_kb_knowledge_knowledge_key", "kb_knowledge", ["knowledge_key"])
|
||||
op.create_table("kb_observations", Column("id", Integer, primary_key=True), Column("kb_product_id", Integer, ForeignKey("kb_products.id"), nullable=False), Column("tenant_id", Integer, ForeignKey("tenants.id"), nullable=False), Column("company_key", String(64), nullable=False), Column("erp_code", String(32), nullable=False), Column("database_label", String(64), nullable=False), Column("observed_fields", JSON, nullable=False), Column("source", String(32), nullable=False, server_default="ERP_OBSERVATION"), Column("observed_at", DateTime(timezone=True), server_default=func.now()))
|
||||
op.create_index("ix_kb_observations_kb_product_id", "kb_observations", ["kb_product_id"])
|
||||
op.create_table("tenant_product_context", Column("id", Integer, primary_key=True), Column("tenant_id", Integer, ForeignKey("tenants.id"), nullable=False), Column("company_key", String(64), nullable=False), Column("erp_code", String(32), nullable=False), Column("kb_product_id", Integer, ForeignKey("kb_products.id")), Column("product_role", String(16)), Column("st_withheld", String(4)), Column("operation_profile", JSON, nullable=False), Column("source", String(32), nullable=False, server_default="ERP_OBSERVATION"), Column("confidence", Float, nullable=False, server_default="0"), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("updated_at", DateTime(timezone=True), server_default=func.now()), UniqueConstraint("tenant_id", "company_key", "erp_code", name="uq_tenant_product_context"))
|
||||
op.create_index("ix_tenant_product_context_tenant_id", "tenant_product_context", ["tenant_id"])
|
||||
op.create_table("ai_work_items", Column("id", Integer, primary_key=True), Column("task_type", String(24), nullable=False), Column("dedup_key", String(160), nullable=False, unique=True), Column("question_refs", JSON, nullable=False), Column("payload", JSON, nullable=False), Column("status", String(24), nullable=False, server_default="PENDING"), Column("kb_product_id", Integer, ForeignKey("kb_products.id")), Column("cells_covered", Integer, nullable=False, server_default="0"), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("updated_at", DateTime(timezone=True), server_default=func.now()))
|
||||
op.create_index("ix_ai_work_items_task_type", "ai_work_items", ["task_type"])
|
||||
op.create_index("ix_ai_work_items_status", "ai_work_items", ["status"])
|
||||
op.create_table("ai_runs", Column("id", Integer, primary_key=True), Column("provider", String(64), nullable=False), Column("model", String(64), nullable=False), Column("prompt_version", String(32), nullable=False), Column("work_item_ids", JSON, nullable=False), Column("input_tokens", Integer, nullable=False, server_default="0"), Column("output_tokens", Integer, nullable=False, server_default="0"), Column("cost_usd", Float, nullable=False, server_default="0"), Column("status", String(16), nullable=False, server_default="RUN"), Column("results", JSON, nullable=False), Column("budget_guard", JSON, nullable=False), Column("started_at", DateTime(timezone=True), server_default=func.now()), Column("finished_at", DateTime(timezone=True)))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("ai_runs")
|
||||
op.drop_table("ai_work_items")
|
||||
op.drop_table("tenant_product_context")
|
||||
op.drop_table("kb_observations")
|
||||
op.drop_table("kb_knowledge")
|
||||
op.drop_index("ix_kb_products_gtin", table_name="kb_products")
|
||||
op.drop_table("kb_products")
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Migration 0013: widen ai_work_items.status for V2 classification labels.
|
||||
|
||||
Revision ID: 0013
|
||||
Revises: 0012
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0013"
|
||||
down_revision = "0012"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.alter_column("ai_work_items", "status", existing_type=sa.String(24), type_=sa.String(32), nullable=False, existing_server_default="PENDING")
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.alter_column("ai_work_items", "status", existing_type=sa.String(32), type_=sa.String(24), nullable=False, existing_server_default="PENDING")
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Migration 0014: persistent LLM eval result cache.
|
||||
|
||||
Revision ID: 0014
|
||||
Revises: 0013
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, DateTime, ForeignKey, Integer, JSON, String, UniqueConstraint, func
|
||||
|
||||
revision = "0014"
|
||||
down_revision = "0013"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table("ai_eval_cache", Column("id", Integer, primary_key=True), Column("input_fingerprint", String(64), nullable=False), Column("candidate_set_hash", String(64), nullable=False), Column("prompt_version", String(32), nullable=False), Column("model", String(64), nullable=False), Column("dataset_version_ncm", String(64), nullable=False), Column("phase", String(16), nullable=False), Column("answer", JSON, nullable=False), Column("gold_ncm", String(8)), Column("created_at", DateTime(timezone=True), server_default=func.now()), UniqueConstraint("input_fingerprint", "candidate_set_hash", "prompt_version", "model", "dataset_version_ncm", name="uq_ai_eval_cache"))
|
||||
op.create_index("ix_ai_eval_cache_input_fingerprint", "ai_eval_cache", ["input_fingerprint"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("ai_eval_cache")
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Add central AI usage ledger and external GTIN cache."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "0015_ai_budget_and_product_cache"
|
||||
down_revision = "0014"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"ai_usage_ledger",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("provider", sa.String(length=64), nullable=False),
|
||||
sa.Column("model", sa.String(length=64), nullable=False),
|
||||
sa.Column("run_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("tenant", sa.String(length=128), nullable=False),
|
||||
sa.Column("items", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("input_tokens", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("output_tokens", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("cost", sa.Float(), nullable=False, server_default="0"),
|
||||
sa.Column("timestamp", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
)
|
||||
op.create_index("ix_ai_usage_ledger_run_id", "ai_usage_ledger", ["run_id"])
|
||||
op.create_index("ix_ai_usage_ledger_tenant", "ai_usage_ledger", ["tenant"])
|
||||
op.create_index("ix_ai_usage_ledger_timestamp", "ai_usage_ledger", ["timestamp"])
|
||||
op.create_table(
|
||||
"external_product_cache",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("gtin", sa.String(length=16), nullable=False),
|
||||
sa.Column("provider", sa.String(length=64), nullable=False),
|
||||
sa.Column("provider_record_version", sa.String(length=64), nullable=False),
|
||||
sa.Column("retrieved_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("raw_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("normalized_data", sa.JSON(), nullable=False),
|
||||
sa.Column("cache_allowed", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("redistribution_allowed", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.UniqueConstraint("gtin", "provider", "provider_record_version", name="uq_external_product_cache_record"),
|
||||
)
|
||||
op.create_index("ix_external_product_cache_gtin", "external_product_cache", ["gtin"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index("ix_external_product_cache_gtin", table_name="external_product_cache")
|
||||
op.drop_table("external_product_cache")
|
||||
op.drop_index("ix_ai_usage_ledger_timestamp", table_name="ai_usage_ledger")
|
||||
op.drop_index("ix_ai_usage_ledger_tenant", table_name="ai_usage_ledger")
|
||||
op.drop_index("ix_ai_usage_ledger_run_id", table_name="ai_usage_ledger")
|
||||
op.drop_table("ai_usage_ledger")
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Add deduplicated blocked product registry."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "0016_blocked_product_registry"
|
||||
down_revision = "0015_ai_budget_and_product_cache"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"blocked_products",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False),
|
||||
sa.Column("company", sa.String(length=128), nullable=False),
|
||||
sa.Column("product_code", sa.String(length=64), nullable=False),
|
||||
sa.Column("gtin", sa.String(length=16)),
|
||||
sa.Column("description", sa.Text()),
|
||||
sa.Column("blocker_type", sa.String(length=48), nullable=False),
|
||||
sa.Column("field", sa.String(length=64), nullable=False),
|
||||
sa.Column("reason_code", sa.String(length=96), nullable=False),
|
||||
sa.Column("evidence_summary", sa.JSON(), nullable=False),
|
||||
sa.Column("input_fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("blocker_fingerprint", sa.String(length=64), nullable=False),
|
||||
sa.Column("first_seen_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("retry_count", sa.Integer(), server_default="0", nullable=False),
|
||||
sa.Column("next_retry_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("resolved_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("resolution_source", sa.String(length=48)),
|
||||
sa.UniqueConstraint("blocker_fingerprint", name="uq_blocked_product_fingerprint"),
|
||||
)
|
||||
for name, column in (("tenant", "tenant_id"), ("company", "company"), ("product", "product_code"), ("gtin", "gtin"), ("blocker_type", "blocker_type"), ("field", "field"), ("input", "input_fingerprint")):
|
||||
op.create_index(f"ix_blocked_products_{name}", "blocked_products", [column])
|
||||
|
||||
|
||||
def downgrade():
|
||||
for name in ("input", "field", "blocker_type", "gtin", "product", "company", "tenant"):
|
||||
op.drop_index(f"ix_blocked_products_{name}", table_name="blocked_products")
|
||||
op.drop_table("blocked_products")
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Add the operational control-plane records."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0017_control_plane_foundation"
|
||||
down_revision = "0016_blocked_product_registry"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"installations",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("agent_id", sa.Integer(), sa.ForeignKey("agents.id"), nullable=False),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False),
|
||||
sa.Column("installation_id", sa.String(36), nullable=False),
|
||||
sa.Column("display_name", sa.String(160), nullable=False, server_default=""),
|
||||
sa.Column("customer_code", sa.String(64)), sa.Column("notes", sa.Text()),
|
||||
sa.Column("hostname", sa.String(255)), sa.Column("windows_version", sa.String(128)),
|
||||
sa.Column("machine_fingerprint", sa.String(128)), sa.Column("company_code", sa.String(64)),
|
||||
sa.Column("company_name", sa.String(160)), sa.Column("company_cnpj_masked", sa.String(24)),
|
||||
sa.Column("company_uf", sa.String(2)), sa.Column("company_crt", sa.String(16)),
|
||||
sa.Column("firebird_version", sa.String(64)), sa.Column("firebird_ods", sa.String(32)),
|
||||
sa.Column("database_identifier", sa.String(255)), sa.Column("capabilities", sa.JSON(), nullable=False, server_default="{}"),
|
||||
sa.Column("fiscal_engine_version", sa.String(64)), sa.Column("dataset_versions", sa.JSON(), nullable=False, server_default="{}"),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True)), sa.Column("last_seen_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_status", sa.String(32)), sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("installation_id", name="uq_installation_identity"),
|
||||
sa.UniqueConstraint("agent_id", name="uq_installation_agent"),
|
||||
)
|
||||
op.create_index("ix_installations_agent_id", "installations", ["agent_id"])
|
||||
op.create_index("ix_installations_tenant_id", "installations", ["tenant_id"])
|
||||
op.create_index("ix_installations_hostname", "installations", ["hostname"])
|
||||
op.create_index("ix_installations_customer_code", "installations", ["customer_code"])
|
||||
op.create_index("ix_installations_last_seen_at", "installations", ["last_seen_at"])
|
||||
op.create_index("ix_installations_installation_id", "installations", ["installation_id"])
|
||||
|
||||
op.create_table(
|
||||
"agent_snapshots",
|
||||
sa.Column("id", sa.Integer(), primary_key=True), sa.Column("installation_id", sa.Integer(), sa.ForeignKey("installations.id"), nullable=False),
|
||||
sa.Column("total_products", sa.Integer(), nullable=False, server_default="0"), sa.Column("fiscally_complete", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("safe_blocked", sa.Integer(), nullable=False, server_default="0"), sa.Column("failed_products", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("coverage_percent", sa.Float(), nullable=False, server_default="0"), sa.Column("blockers", sa.JSON(), nullable=False, server_default="{}"),
|
||||
sa.Column("last_job_status", sa.String(16)), sa.Column("last_job_at", sa.DateTime(timezone=True)), sa.Column("last_successful_job_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("agent_health", sa.String(16), nullable=False, server_default="UNKNOWN"), sa.Column("health_reason", sa.Text()),
|
||||
sa.Column("version_drift", sa.String(16), nullable=False, server_default="UNKNOWN"), sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("installation_id", name="uq_agent_snapshot_installation"),
|
||||
)
|
||||
for name, column in (("installation", "installation_id"), ("health", "agent_health"), ("version_drift", "version_drift"), ("updated", "updated_at")):
|
||||
op.create_index(f"ix_agent_snapshots_{name}", "agent_snapshots", [column])
|
||||
|
||||
op.create_table(
|
||||
"agent_events",
|
||||
sa.Column("id", sa.Integer(), primary_key=True), sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False),
|
||||
sa.Column("installation_id", sa.Integer(), sa.ForeignKey("installations.id"), nullable=False), sa.Column("agent_id", sa.Integer(), sa.ForeignKey("agents.id"), nullable=False),
|
||||
sa.Column("severity", sa.String(16), nullable=False), sa.Column("event_type", sa.String(48), nullable=False), sa.Column("message", sa.Text(), nullable=False),
|
||||
sa.Column("structured_context", sa.JSON(), nullable=False, server_default="{}"), sa.Column("job_id", sa.Integer(), sa.ForeignKey("jobs.id")),
|
||||
sa.Column("product_code", sa.String(64)), sa.Column("blocker_id", sa.Integer(), sa.ForeignKey("blocked_products.id")), sa.Column("is_audit", sa.Boolean(), nullable=False, server_default="false"),
|
||||
sa.Column("occurred_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
for name, column in (("tenant", "tenant_id"), ("installation", "installation_id"), ("agent", "agent_id"), ("severity", "severity"), ("event_type", "event_type"), ("job", "job_id"), ("product", "product_code"), ("blocker", "blocker_id"), ("audit", "is_audit"), ("occurred", "occurred_at")):
|
||||
op.create_index(f"ix_agent_events_{name}", "agent_events", [column])
|
||||
|
||||
op.create_table(
|
||||
"admin_users",
|
||||
sa.Column("id", sa.Integer(), primary_key=True), sa.Column("username", sa.String(128), nullable=False, unique=True),
|
||||
sa.Column("password_hash", sa.String(255), nullable=False), sa.Column("is_active", sa.Boolean(), nullable=False, server_default="true"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), sa.Column("last_login_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
op.create_index("ix_admin_users_username", "admin_users", ["username"])
|
||||
op.create_table(
|
||||
"web_sessions",
|
||||
sa.Column("id", sa.Integer(), primary_key=True), sa.Column("admin_user_id", sa.Integer(), sa.ForeignKey("admin_users.id"), nullable=False),
|
||||
sa.Column("token_hash", sa.String(64), nullable=False, unique=True), sa.Column("csrf_hash", sa.String(64), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), server_default=sa.func.now()), sa.Column("user_agent", sa.String(512)), sa.Column("remote_addr", sa.String(64)),
|
||||
)
|
||||
op.create_index("ix_web_sessions_admin_user_id", "web_sessions", ["admin_user_id"])
|
||||
op.create_index("ix_web_sessions_token_hash", "web_sessions", ["token_hash"])
|
||||
op.create_index("ix_web_sessions_expires_at", "web_sessions", ["expires_at"])
|
||||
op.create_table(
|
||||
"audit_events",
|
||||
sa.Column("id", sa.Integer(), primary_key=True), sa.Column("admin_user_id", sa.Integer(), sa.ForeignKey("admin_users.id")),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id")), sa.Column("action", sa.String(64), nullable=False),
|
||||
sa.Column("target_type", sa.String(64), nullable=False), sa.Column("target_id", sa.String(128), nullable=False), sa.Column("context", sa.JSON(), nullable=False, server_default="{}"),
|
||||
sa.Column("severity", sa.String(16), nullable=False, server_default="INFO"), sa.Column("occurred_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
for name, column in (("admin", "admin_user_id"), ("tenant", "tenant_id"), ("action", "action"), ("occurred", "occurred_at")):
|
||||
op.create_index(f"ix_audit_events_{name}", "audit_events", [column])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("audit_events")
|
||||
op.drop_table("web_sessions")
|
||||
op.drop_table("admin_users")
|
||||
op.drop_table("agent_events")
|
||||
op.drop_table("agent_snapshots")
|
||||
op.drop_table("installations")
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Associate new blocker rows with their authenticated installation."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "0018_blocker_installation"
|
||||
down_revision = "0017_control_plane_foundation"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("blocked_products", sa.Column("installation_id", sa.Integer(), sa.ForeignKey("installations.id"), nullable=True))
|
||||
op.create_index("ix_blocked_products_installation_id", "blocked_products", ["installation_id"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index("ix_blocked_products_installation_id", table_name="blocked_products")
|
||||
op.drop_column("blocked_products", "installation_id")
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Fix frozen timestamp defaults from 0006 and add model indexes never migrated.
|
||||
|
||||
0006 declared ``server_default="now()"`` as a plain string. SQLAlchemy renders
|
||||
that as the literal ``'now()'``, which PostgreSQL casts to a timestamp once,
|
||||
when the column is created — every later row got that same fixed instant
|
||||
instead of the insert time (agents.created_at/updated_at, jobs.created_at).
|
||||
The models rely on the server default, so Jobs and Agents registered after
|
||||
0006 all carried the migration's timestamp. Existing rows are left as they
|
||||
are: their real creation time is not recoverable.
|
||||
|
||||
The models also declare ``index=True`` on three columns that no migration
|
||||
ever indexed.
|
||||
|
||||
Revision ID: 0019_fix_timestamp_defaults
|
||||
Revises: 0018_blocker_installation
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0019_fix_timestamp_defaults"
|
||||
down_revision = "0018_blocker_installation"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TIMESTAMP_COLUMNS = (("agents", "created_at"), ("agents", "updated_at"), ("jobs", "created_at"))
|
||||
_MISSING_INDEXES = (
|
||||
("ix_kb_observations_tenant_id", "kb_observations", "tenant_id"),
|
||||
("ix_kb_observations_company_key", "kb_observations", "company_key"),
|
||||
("ix_tenant_product_context_company_key", "tenant_product_context", "company_key"),
|
||||
)
|
||||
|
||||
|
||||
def upgrade():
|
||||
for table, column in _TIMESTAMP_COLUMNS:
|
||||
op.alter_column(table, column, server_default=sa.func.now(), existing_type=sa.DateTime(timezone=True), existing_nullable=False)
|
||||
for name, table, column in _MISSING_INDEXES:
|
||||
op.create_index(name, table, [column])
|
||||
|
||||
|
||||
def downgrade():
|
||||
# The frozen-timestamp defaults are a bug and are deliberately not restored.
|
||||
for name, table, _column in reversed(_MISSING_INDEXES):
|
||||
op.drop_index(name, table_name=table)
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Enforce NOT NULL on server-defaulted timestamps the models already require.
|
||||
|
||||
0011, 0012, 0014 and 0017 created these columns with ``server_default=now()``
|
||||
but without ``nullable=False``, so the database accepted NULL while the models
|
||||
declare them non-optional. No application path writes NULL to them (inserts
|
||||
omit them and rely on the default; updates set ``now``), and before this
|
||||
revision was written no NULL existed in production or in the development
|
||||
database.
|
||||
|
||||
The upgrade refuses to run — changing nothing — if any NULL is found, instead
|
||||
of guessing a value for it.
|
||||
|
||||
Revision ID: 0020_not_null_timestamps
|
||||
Revises: 0019_fix_timestamp_defaults
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0020_not_null_timestamps"
|
||||
down_revision = "0019_fix_timestamp_defaults"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
COLUMNS = (
|
||||
("admin_users", "created_at"),
|
||||
("agent_events", "occurred_at"),
|
||||
("agent_snapshots", "updated_at"),
|
||||
("ai_eval_cache", "created_at"),
|
||||
("ai_runs", "started_at"),
|
||||
("ai_work_items", "created_at"),
|
||||
("ai_work_items", "updated_at"),
|
||||
("audit_events", "occurred_at"),
|
||||
("fiscal_context_entries", "created_at"),
|
||||
("fiscal_context_entries", "updated_at"),
|
||||
("installations", "created_at"),
|
||||
("installations", "updated_at"),
|
||||
("kb_knowledge", "created_at"),
|
||||
("kb_observations", "observed_at"),
|
||||
("kb_products", "created_at"),
|
||||
("kb_products", "updated_at"),
|
||||
("tenant_product_context", "created_at"),
|
||||
("tenant_product_context", "updated_at"),
|
||||
("web_sessions", "created_at"),
|
||||
("web_sessions", "last_seen_at"),
|
||||
)
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
offenders = []
|
||||
for table, column in COLUMNS:
|
||||
nulls = bind.execute(sa.text(f'SELECT count(*) FROM "{table}" WHERE "{column}" IS NULL')).scalar()
|
||||
if nulls:
|
||||
offenders.append(f"{table}.{column}={nulls}")
|
||||
if offenders:
|
||||
raise RuntimeError("0020 aborted, NULL values present (fix them explicitly first): " + ", ".join(offenders))
|
||||
for table, column in COLUMNS:
|
||||
op.alter_column(table, column, nullable=False, existing_type=sa.DateTime(timezone=True), existing_server_default=sa.func.now())
|
||||
|
||||
|
||||
def downgrade():
|
||||
for table, column in reversed(COLUMNS):
|
||||
op.alter_column(table, column, nullable=True, existing_type=sa.DateTime(timezone=True), existing_server_default=sa.func.now())
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Federated identity for SSO from GTI Gerenciador.
|
||||
|
||||
admin_users stays the single human principal (web_sessions and audit_events
|
||||
already point at it). Existing rows become auth_source='LOCAL' (the
|
||||
break-glass admin keeps its password). Collaborators provisioned by SSO are
|
||||
auth_source='GTI_GERENCIADOR': no password at all, identified by the panel's
|
||||
immutable subject. The CHECK constraint makes a passwordless LOCAL account,
|
||||
or a federated account with a password, impossible.
|
||||
|
||||
sso_used_assertions keeps every accepted assertion id (jti) so the same
|
||||
assertion can never be used twice.
|
||||
|
||||
Revision ID: 0021_sso_federated_identity
|
||||
Revises: 0020_not_null_timestamps
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0021_sso_federated_identity"
|
||||
down_revision = "0020_not_null_timestamps"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
AUTH_SOURCE_CHECK = (
|
||||
"(auth_source = 'LOCAL' AND password_hash IS NOT NULL AND external_subject IS NULL)"
|
||||
" OR (auth_source = 'GTI_GERENCIADOR' AND password_hash IS NULL AND external_subject IS NOT NULL)"
|
||||
)
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("admin_users", sa.Column("auth_source", sa.String(16), nullable=False, server_default="LOCAL"))
|
||||
op.add_column("admin_users", sa.Column("external_subject", sa.String(128), nullable=True))
|
||||
op.add_column("admin_users", sa.Column("display_name", sa.String(160), nullable=True))
|
||||
op.add_column("admin_users", sa.Column("external_role", sa.String(32), nullable=True))
|
||||
op.add_column("admin_users", sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False))
|
||||
op.alter_column("admin_users", "password_hash", existing_type=sa.String(255), nullable=True)
|
||||
op.create_unique_constraint("uq_admin_users_external_subject", "admin_users", ["external_subject"])
|
||||
op.create_check_constraint("ck_admin_users_auth_source", "admin_users", AUTH_SOURCE_CHECK)
|
||||
op.create_table(
|
||||
"sso_used_assertions",
|
||||
sa.Column("jti", sa.String(64), primary_key=True),
|
||||
sa.Column("subject", sa.String(128), nullable=False),
|
||||
sa.Column("used_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_sso_used_assertions_expires_at", "sso_used_assertions", ["expires_at"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
if bind.execute(sa.text("SELECT count(*) FROM admin_users WHERE auth_source <> 'LOCAL'")).scalar():
|
||||
raise RuntimeError("0021 downgrade refused: federated users exist (their sessions/audit rows reference them)")
|
||||
op.drop_index("ix_sso_used_assertions_expires_at", table_name="sso_used_assertions")
|
||||
op.drop_table("sso_used_assertions")
|
||||
op.drop_constraint("ck_admin_users_auth_source", "admin_users", type_="check")
|
||||
op.drop_constraint("uq_admin_users_external_subject", "admin_users", type_="unique")
|
||||
op.alter_column("admin_users", "password_hash", existing_type=sa.String(255), nullable=False)
|
||||
for column in ("updated_at", "external_role", "display_name", "external_subject", "auth_source"):
|
||||
op.drop_column("admin_users", column)
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Agent distribution: releases + secure enrollment.
|
||||
|
||||
agent_releases catalogs published builds of the universal Windows Agent
|
||||
package (the binary itself lives in local release storage, never in
|
||||
Postgres). agent_enrollments tracks one pending-or-completed installation
|
||||
per row, carrying two distinct hashed, single-use, short-TTL secrets: a
|
||||
download token (the shareable package link) and an enrollment token
|
||||
(embedded in the downloaded bundle, consumed once by POST /v1/agents/enroll).
|
||||
Neither secret is ever stored in plaintext.
|
||||
|
||||
Revision ID: 0022_agent_releases_enrollment
|
||||
Revises: 0021_sso_federated_identity
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0022_agent_releases_enrollment"
|
||||
down_revision = "0021_sso_federated_identity"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"agent_releases",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("version", sa.String(32), nullable=False),
|
||||
sa.Column("platform", sa.String(16), nullable=False, server_default="windows"),
|
||||
sa.Column("architecture", sa.String(16), nullable=False, server_default="x64"),
|
||||
sa.Column("filename", sa.String(255), nullable=False),
|
||||
sa.Column("size_bytes", sa.Integer(), nullable=False),
|
||||
sa.Column("sha256", sa.String(64), nullable=False),
|
||||
sa.Column("storage_path", sa.String(512), nullable=False),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="DRAFT"),
|
||||
sa.Column("release_notes", sa.Text(), nullable=True),
|
||||
sa.Column("created_by", sa.Integer(), sa.ForeignKey("admin_users.id"), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("published_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.UniqueConstraint("version", "platform", "architecture", name="uq_agent_release_version"),
|
||||
)
|
||||
op.create_index("ix_agent_releases_version", "agent_releases", ["version"])
|
||||
op.create_index("ix_agent_releases_sha256", "agent_releases", ["sha256"])
|
||||
op.create_index("ix_agent_releases_status", "agent_releases", ["status"])
|
||||
|
||||
op.create_table(
|
||||
"agent_enrollments",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("enrollment_uuid", sa.String(36), nullable=False),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False),
|
||||
sa.Column("display_name", sa.String(160), nullable=False),
|
||||
sa.Column("expected_hostname", sa.String(255), nullable=True),
|
||||
sa.Column("release_id", sa.Integer(), sa.ForeignKey("agent_releases.id"), nullable=False),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="PENDING"),
|
||||
sa.Column("created_by", sa.Integer(), sa.ForeignKey("admin_users.id"), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("download_token_hash", sa.String(64), nullable=True),
|
||||
sa.Column("download_token_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("download_max_uses", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("download_uses", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("downloaded_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("enrollment_token_hash", sa.String(64), nullable=True),
|
||||
sa.Column("enrollment_token_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("enrollment_used_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("machine_fingerprint_seen", sa.String(128), nullable=True),
|
||||
sa.Column("hostname_seen", sa.String(255), nullable=True),
|
||||
sa.Column("hostname_mismatch", sa.Boolean(), nullable=False, server_default="false"),
|
||||
sa.Column("agent_id", sa.Integer(), sa.ForeignKey("agents.id"), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.UniqueConstraint("enrollment_uuid", name="uq_agent_enrollment_uuid"),
|
||||
)
|
||||
op.create_index("ix_agent_enrollments_uuid", "agent_enrollments", ["enrollment_uuid"])
|
||||
op.create_index("ix_agent_enrollments_tenant", "agent_enrollments", ["tenant_id"])
|
||||
op.create_index("ix_agent_enrollments_release", "agent_enrollments", ["release_id"])
|
||||
op.create_index("ix_agent_enrollments_status", "agent_enrollments", ["status"])
|
||||
op.create_index("ix_agent_enrollments_agent", "agent_enrollments", ["agent_id"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index("ix_agent_enrollments_agent", table_name="agent_enrollments")
|
||||
op.drop_index("ix_agent_enrollments_status", table_name="agent_enrollments")
|
||||
op.drop_index("ix_agent_enrollments_release", table_name="agent_enrollments")
|
||||
op.drop_index("ix_agent_enrollments_tenant", table_name="agent_enrollments")
|
||||
op.drop_index("ix_agent_enrollments_uuid", table_name="agent_enrollments")
|
||||
op.drop_table("agent_enrollments")
|
||||
op.drop_index("ix_agent_releases_status", table_name="agent_releases")
|
||||
op.drop_index("ix_agent_releases_sha256", table_name="agent_releases")
|
||||
op.drop_index("ix_agent_releases_version", table_name="agent_releases")
|
||||
op.drop_table("agent_releases")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Installation code (install_code) for the Windows installer EXE flow.
|
||||
|
||||
agent_enrollments gains the install-code columns used by the universal EXE
|
||||
installer: the code is typed by the operator into Setup.exe and resolves the
|
||||
installation on the Hub. Only the SHA-256 of the normalized code is ever
|
||||
stored (install_code_hash); install_code_expires_at bounds its life and
|
||||
install_code_created_at records when it was generated (rotation rewrites
|
||||
these). install_code_attempts counts failed enrollment tries to support the
|
||||
per-installation rate limit. No plaintext code column exists.
|
||||
|
||||
Revision ID: 0023_install_codes
|
||||
Revises: 0022_agent_releases_enrollment
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0023_install_codes"
|
||||
down_revision = "0022_agent_releases_enrollment"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("agent_enrollments", sa.Column("install_code_hash", sa.String(64), nullable=True))
|
||||
op.add_column("agent_enrollments", sa.Column("install_code_expires_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("agent_enrollments", sa.Column("install_code_attempts", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.add_column("agent_enrollments", sa.Column("install_code_created_at", sa.DateTime(timezone=True), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("agent_enrollments", "install_code_created_at")
|
||||
op.drop_column("agent_enrollments", "install_code_attempts")
|
||||
op.drop_column("agent_enrollments", "install_code_expires_at")
|
||||
op.drop_column("agent_enrollments", "install_code_hash")
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Tenant mirror identity from GTI Gerenciador.
|
||||
|
||||
GTI Gerenciador (the panel) becomes the source of truth for clients.
|
||||
tenants gains source ('LOCAL' | 'GTI_GERENCIADOR'), external_client_id
|
||||
(Gerenciador's CLIENTE.ID), external_code, active and synced_at. Existing
|
||||
rows (pre-dating this sync) become source='LOCAL' and are never offered for
|
||||
a new installation, but keep every foreign key that already points at them.
|
||||
|
||||
Revision ID: 0024_tenant_gerenciador_mirror
|
||||
Revises: 0023_install_codes
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0024_tenant_gerenciador_mirror"
|
||||
down_revision = "0023_install_codes"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("tenants", sa.Column("source", sa.String(16), nullable=False, server_default="LOCAL"))
|
||||
op.add_column("tenants", sa.Column("external_client_id", sa.Integer(), nullable=True))
|
||||
op.add_column("tenants", sa.Column("external_code", sa.String(16), nullable=True))
|
||||
op.add_column("tenants", sa.Column("active", sa.Boolean(), nullable=False, server_default="true"))
|
||||
op.add_column("tenants", sa.Column("synced_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.create_index("ix_tenants_external_client_id", "tenants", ["external_client_id"])
|
||||
op.create_unique_constraint("uq_tenant_external_identity", "tenants", ["source", "external_client_id"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_constraint("uq_tenant_external_identity", "tenants", type_="unique")
|
||||
op.drop_index("ix_tenants_external_client_id", table_name="tenants")
|
||||
for column in ("synced_at", "active", "external_code", "external_client_id", "source"):
|
||||
op.drop_column("tenants", column)
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Tenant.external_cnpj (digits) mirrored from GTI Gerenciador for the client selector search.
|
||||
|
||||
Revision ID: 0025_tenant_external_cnpj
|
||||
Revises: 0024_tenant_gerenciador_mirror
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0025_tenant_external_cnpj"
|
||||
down_revision = "0024_tenant_gerenciador_mirror"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("tenants", sa.Column("external_cnpj", sa.String(20), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("tenants", "external_cnpj")
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Central processing control + sync bookkeeping per installation.
|
||||
|
||||
Revision ID: 0026_central_processing_control
|
||||
Revises: 0025_tenant_external_cnpj
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0026_central_processing_control"
|
||||
down_revision = "0025_tenant_external_cnpj"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("installations", sa.Column("processing_enabled", sa.Boolean(), nullable=False, server_default="true"))
|
||||
op.add_column("installations", sa.Column("sync_interval_seconds", sa.Integer(), nullable=False, server_default="1800"))
|
||||
op.add_column("installations", sa.Column("last_sync_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("installations", sa.Column("last_sync_changed", sa.Integer(), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
for column in ("last_sync_changed", "last_sync_at", "sync_interval_seconds", "processing_enabled"):
|
||||
op.drop_column("installations", column)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""RTC auto-apply: per-installation authorization (default OFF) + immutable write audit.
|
||||
|
||||
Revision ID: 0027_rtc_auto_apply
|
||||
Revises: 0026_central_processing_control
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0027_rtc_auto_apply"
|
||||
down_revision = "0026_central_processing_control"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("installations", sa.Column("rtc_auto_apply_authorized", sa.Boolean(), nullable=False, server_default="false"))
|
||||
op.create_table(
|
||||
"rtc_write_audits",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False, index=True),
|
||||
sa.Column("installation_id", sa.Integer(), sa.ForeignKey("installations.id"), nullable=False, index=True),
|
||||
sa.Column("agent_id", sa.Integer(), sa.ForeignKey("agents.id"), nullable=False, index=True),
|
||||
sa.Column("job_id", sa.Integer(), nullable=True, index=True),
|
||||
sa.Column("client", sa.String(255), nullable=True),
|
||||
sa.Column("product_code", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("field", sa.String(32), nullable=False),
|
||||
sa.Column("outcome", sa.String(16), nullable=False, index=True),
|
||||
sa.Column("reason", sa.String(96), nullable=True),
|
||||
sa.Column("old_value", sa.String(64), nullable=True),
|
||||
sa.Column("new_value", sa.String(64), nullable=True),
|
||||
sa.Column("rule_id", sa.String(32), nullable=True),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False, server_default="{}"),
|
||||
sa.Column("official_source_version", sa.JSON(), nullable=False, server_default="{}"),
|
||||
sa.Column("local_catalog_hash", sa.String(64), nullable=True),
|
||||
sa.Column("written_at", sa.String(40), nullable=True),
|
||||
sa.Column("recorded_at", sa.DateTime(timezone=True), server_default=sa.func.now(), index=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("rtc_write_audits")
|
||||
op.drop_column("installations", "rtc_auto_apply_authorized")
|
||||
@@ -0,0 +1,47 @@
|
||||
"""RTC central write modes per installation, candidate preview/approval table, richer audit + rollback tracking.
|
||||
|
||||
Revision ID: 0028_rtc_write_modes
|
||||
Revises: 0027_rtc_auto_apply
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0028_rtc_write_modes"
|
||||
down_revision = "0027_rtc_auto_apply"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("installations", sa.Column("rtc_write_mode", sa.String(24), nullable=False, server_default="DISABLED"))
|
||||
op.execute("UPDATE installations SET rtc_write_mode = 'ENABLED' WHERE rtc_auto_apply_authorized") # keeps an existing explicit authorization
|
||||
for name, kind in (("rule_version", sa.String(32)), ("company_crt", sa.String(8)), ("authorization_origin", sa.String(48)), ("description_snapshot", sa.String(255)),
|
||||
("installation_uuid", sa.String(64)), ("rtc_status", sa.String(24)), ("action", sa.String(16)), ("rollback_state", sa.String(16)), ("rollback_of", sa.Integer()),
|
||||
("local_catalog_state", sa.String(32))):
|
||||
op.add_column("rtc_write_audits", sa.Column(name, kind, nullable=True))
|
||||
op.create_index("ix_rtc_write_audits_rollback_of", "rtc_write_audits", ["rollback_of"])
|
||||
op.create_table(
|
||||
"rtc_candidates",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False, index=True),
|
||||
sa.Column("installation_id", sa.Integer(), sa.ForeignKey("installations.id"), nullable=False, index=True),
|
||||
sa.Column("product_code", sa.String(64), nullable=False),
|
||||
sa.Column("description", sa.String(255), nullable=True),
|
||||
sa.Column("ncm", sa.String(16), nullable=True),
|
||||
sa.Column("current_value", sa.String(16), nullable=True),
|
||||
sa.Column("proposed", sa.String(16), nullable=False),
|
||||
sa.Column("cst_ibscbs", sa.String(8), nullable=True),
|
||||
sa.Column("rule_id", sa.String(32), nullable=False),
|
||||
sa.Column("basis", sa.String(255), nullable=True),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="PENDING", index=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("installation_id", "product_code", name="uq_rtc_candidate_product"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("rtc_candidates")
|
||||
op.drop_index("ix_rtc_write_audits_rollback_of", table_name="rtc_write_audits")
|
||||
for name in ("rule_version", "company_crt", "authorization_origin", "description_snapshot", "installation_uuid", "rtc_status", "action", "rollback_state", "rollback_of", "local_catalog_state"):
|
||||
op.drop_column("rtc_write_audits", name)
|
||||
op.drop_column("installations", "rtc_write_mode")
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Per-installation functions (corrector / enrichment mode), KB field status, enrichment state + audit, provider state/usage/cache, official source refresh log.
|
||||
|
||||
Revision ID: 0029_product_enrichment
|
||||
Revises: 0028_rtc_write_modes
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0029_product_enrichment"
|
||||
down_revision = "0028_rtc_write_modes"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _idx(table, *cols):
|
||||
for col in cols:
|
||||
op.create_index(f"ix_{table}_{col}", table, [col])
|
||||
|
||||
|
||||
def upgrade():
|
||||
# existing installations keep behaving exactly as before: corrector on, functions already configured
|
||||
op.add_column("installations", sa.Column("corrector_enabled", sa.Boolean(), nullable=False, server_default=sa.text("true")))
|
||||
op.add_column("installations", sa.Column("enrichment_mode", sa.String(16), nullable=False, server_default="DISABLED"))
|
||||
op.add_column("installations", sa.Column("features_configured", sa.Boolean(), nullable=False, server_default=sa.text("true")))
|
||||
op.add_column("agent_enrollments", sa.Column("features_selected", sa.Boolean(), nullable=False, server_default=sa.text("false")))
|
||||
op.add_column("agent_enrollments", sa.Column("feature_corrector", sa.Boolean(), nullable=False, server_default=sa.text("false")))
|
||||
op.add_column("agent_enrollments", sa.Column("feature_enrichment_mode", sa.String(16), nullable=False, server_default="DISABLED"))
|
||||
op.add_column("kb_knowledge", sa.Column("status", sa.String(16), nullable=False, server_default="EVIDENCE_ONLY"))
|
||||
op.create_index("ix_kb_knowledge_status", "kb_knowledge", ["status"])
|
||||
|
||||
op.create_table(
|
||||
"product_enrichments",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False),
|
||||
sa.Column("installation_id", sa.Integer(), sa.ForeignKey("installations.id"), nullable=False),
|
||||
sa.Column("product_code", sa.String(64), nullable=False),
|
||||
sa.Column("gtin", sa.String(16), nullable=True),
|
||||
sa.Column("description", sa.String(255), nullable=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="PENDING"),
|
||||
sa.Column("decisions", sa.JSON(), nullable=False),
|
||||
sa.Column("fields_applied", sa.JSON(), nullable=False),
|
||||
sa.Column("human_overridden", sa.JSON(), nullable=False),
|
||||
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("latency_ms", sa.Integer(), nullable=True),
|
||||
sa.Column("detected_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("installation_id", "product_code", name="uq_product_enrichment"),
|
||||
)
|
||||
_idx("product_enrichments", "tenant_id", "installation_id", "gtin", "status", "next_attempt_at")
|
||||
|
||||
op.create_table(
|
||||
"product_field_audits",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False),
|
||||
sa.Column("installation_id", sa.Integer(), sa.ForeignKey("installations.id"), nullable=False),
|
||||
sa.Column("agent_id", sa.Integer(), sa.ForeignKey("agents.id"), nullable=False),
|
||||
sa.Column("product_code", sa.String(64), nullable=False),
|
||||
sa.Column("gtin", sa.String(16), nullable=True),
|
||||
sa.Column("description_snapshot", sa.String(255), nullable=True),
|
||||
sa.Column("field", sa.String(48), nullable=False),
|
||||
sa.Column("old_value", sa.String(64), nullable=True),
|
||||
sa.Column("new_value", sa.String(64), nullable=True),
|
||||
sa.Column("rule", sa.String(64), nullable=True),
|
||||
sa.Column("kb_status", sa.String(16), nullable=True),
|
||||
sa.Column("evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("dataset_versions", sa.JSON(), nullable=False),
|
||||
sa.Column("authorization_origin", sa.String(48), nullable=True),
|
||||
sa.Column("job_id", sa.Integer(), nullable=True),
|
||||
sa.Column("outcome", sa.String(24), nullable=False, server_default="APPLIED"),
|
||||
sa.Column("reason", sa.String(96), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
_idx("product_field_audits", "tenant_id", "installation_id", "agent_id", "product_code")
|
||||
|
||||
op.create_table(
|
||||
"provider_states",
|
||||
sa.Column("provider", sa.String(32), primary_key=True),
|
||||
sa.Column("configured", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="NOT_CONFIGURED"),
|
||||
sa.Column("quota_kind", sa.String(16), nullable=False, server_default="DOCUMENTED_RATE"),
|
||||
sa.Column("daily_limit", sa.Integer(), nullable=True),
|
||||
sa.Column("remaining", sa.Integer(), nullable=True),
|
||||
sa.Column("reset_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("rate_per_minute", sa.Integer(), nullable=True),
|
||||
sa.Column("circuit_open_until", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("consecutive_failures", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("last_request_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.String(160), nullable=True),
|
||||
sa.Column("last_error_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_latency_ms", sa.Integer(), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
op.create_table(
|
||||
"provider_usage_daily",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("provider", sa.String(32), nullable=False),
|
||||
sa.Column("day", sa.String(10), nullable=False),
|
||||
*[sa.Column(name, sa.Integer(), nullable=False, server_default="0") for name in ("requests", "success", "not_found", "errors", "rate_limited", "timeouts", "cache_hits", "cache_misses")],
|
||||
sa.UniqueConstraint("provider", "day", name="uq_provider_usage_day"),
|
||||
)
|
||||
_idx("provider_usage_daily", "provider", "day")
|
||||
op.create_table(
|
||||
"gtin_lookup_cache",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("gtin", sa.String(16), nullable=False),
|
||||
sa.Column("provider", sa.String(32), nullable=False),
|
||||
sa.Column("found", sa.Boolean(), nullable=False),
|
||||
sa.Column("data", sa.JSON(), nullable=False),
|
||||
sa.Column("fetched_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("gtin", "provider", name="uq_gtin_lookup_cache"),
|
||||
)
|
||||
_idx("gtin_lookup_cache", "gtin", "expires_at")
|
||||
op.create_table(
|
||||
"source_refresh_runs",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("source", sa.String(32), nullable=False),
|
||||
sa.Column("source_url", sa.Text(), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="RUNNING"),
|
||||
sa.Column("sha256", sa.String(64), nullable=True),
|
||||
sa.Column("version_label", sa.String(64), nullable=True),
|
||||
sa.Column("record_count", sa.Integer(), nullable=True),
|
||||
sa.Column("error", sa.String(300), nullable=True),
|
||||
)
|
||||
_idx("source_refresh_runs", "source")
|
||||
|
||||
|
||||
def downgrade():
|
||||
for table in ("source_refresh_runs", "gtin_lookup_cache", "provider_usage_daily", "provider_states", "product_field_audits", "product_enrichments"):
|
||||
op.drop_table(table)
|
||||
op.drop_index("ix_kb_knowledge_status", table_name="kb_knowledge")
|
||||
op.drop_column("kb_knowledge", "status")
|
||||
for name in ("feature_enrichment_mode", "feature_corrector", "features_selected"):
|
||||
op.drop_column("agent_enrollments", name)
|
||||
for name in ("features_configured", "enrichment_mode", "corrector_enabled"):
|
||||
op.drop_column("installations", name)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""KB evidence model: wider per-field status (OBSERVED/CORROBORATED/EXTERNAL_CORROBORATED/...), promotion audit, provider counters for the panel.
|
||||
|
||||
Revision ID: 0030_kb_evidence_model
|
||||
Revises: 0029_product_enrichment
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0030_kb_evidence_model"
|
||||
down_revision = "0029_product_enrichment"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.alter_column("kb_knowledge", "status", existing_type=sa.String(16), type_=sa.String(24), existing_nullable=False, existing_server_default="EVIDENCE_ONLY")
|
||||
op.execute("UPDATE kb_knowledge SET status='OBSERVED' WHERE status='EVIDENCE_ONLY'")
|
||||
op.alter_column("kb_knowledge", "status", server_default="OBSERVED")
|
||||
op.create_table(
|
||||
"kb_promotion_audits",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("kb_product_id", sa.Integer(), sa.ForeignKey("kb_products.id"), nullable=False),
|
||||
sa.Column("gtin", sa.String(16), nullable=True),
|
||||
sa.Column("field", sa.String(24), nullable=False, server_default="ncm"),
|
||||
sa.Column("value", sa.String(64), nullable=True),
|
||||
sa.Column("old_status", sa.String(24), nullable=True),
|
||||
sa.Column("new_status", sa.String(24), nullable=False),
|
||||
sa.Column("families", sa.JSON(), nullable=False),
|
||||
sa.Column("sources", sa.JSON(), nullable=False),
|
||||
sa.Column("official_dataset", sa.JSON(), nullable=False),
|
||||
sa.Column("rule_version", sa.String(32), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_kb_promotion_audits_kb_product_id", "kb_promotion_audits", ["kb_product_id"])
|
||||
op.create_index("ix_kb_promotion_audits_gtin", "kb_promotion_audits", ["gtin"])
|
||||
op.add_column("provider_usage_daily", sa.Column("found", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.add_column("provider_usage_daily", sa.Column("ncm_found", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.add_column("provider_usage_daily", sa.Column("kb_hits", sa.Integer(), nullable=False, server_default="0"))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("provider_usage_daily", "kb_hits")
|
||||
op.drop_column("provider_usage_daily", "ncm_found")
|
||||
op.drop_column("provider_usage_daily", "found")
|
||||
op.drop_table("kb_promotion_audits")
|
||||
op.execute("UPDATE kb_knowledge SET status='EVIDENCE_ONLY' WHERE status IN ('OBSERVED','CORROBORATED','EXTERNAL_CORROBORATED')")
|
||||
op.alter_column("kb_knowledge", "status", server_default="EVIDENCE_ONLY")
|
||||
op.alter_column("kb_knowledge", "status", existing_type=sa.String(24), type_=sa.String(16), existing_nullable=False, existing_server_default="EVIDENCE_ONLY")
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Enrichment re-evaluation + visible per-product status (no silent failures).
|
||||
|
||||
Revision ID: 0031_enrichment_reevaluate
|
||||
Revises: 0030_kb_evidence_model
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0031_enrichment_reevaluate"
|
||||
down_revision = "0030_kb_evidence_model"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("product_enrichments", sa.Column("reevaluate", sa.Boolean(), nullable=False, server_default=sa.text("false")))
|
||||
op.add_column("product_enrichments", sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("product_enrichments", sa.Column("ui_status", sa.String(32), nullable=True))
|
||||
op.add_column("product_enrichments", sa.Column("reason_code", sa.String(48), nullable=True))
|
||||
op.create_index("ix_product_enrichments_reevaluate", "product_enrichments", ["reevaluate"])
|
||||
op.create_index("ix_product_enrichments_ui_status", "product_enrichments", ["ui_status"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index("ix_product_enrichments_ui_status", table_name="product_enrichments")
|
||||
op.drop_index("ix_product_enrichments_reevaluate", table_name="product_enrichments")
|
||||
for name in ("reason_code", "ui_status", "last_attempt_at", "reevaluate"):
|
||||
op.drop_column("product_enrichments", name)
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Installation.company_contribuinte_ipi: auto-discovered (C000004.CONTRIBUINTE_IPI, same row/query as company_crt) at heartbeat, never
|
||||
operator-guessed. Closes the fiscal-completion gap where CST_IPI could not be resolved without asking a human for a fact the ERP already proves.
|
||||
|
||||
Revision ID: 0032_contribuinte_ipi
|
||||
Revises: 0031_enrichment_reevaluate
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0032_contribuinte_ipi"
|
||||
down_revision = "0031_enrichment_reevaluate"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("installations", sa.Column("company_contribuinte_ipi", sa.String(1), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("installations", "company_contribuinte_ipi")
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Reference Data Registry (Missão 5, Fase 1): additive columns on datasets for licence terms,
|
||||
vigência, file provenance, parse errors and admin approval tracking. Existing rows/kinds are
|
||||
untouched - new nullable columns, no data migration, no behavior change for the 9 pre-existing
|
||||
dataset kinds (their own service.py ingestion methods still auto-activate exactly as before).
|
||||
|
||||
Revision ID: 0034_reference_data_registry
|
||||
Revises: 0032_contribuinte_ipi
|
||||
|
||||
Note: migration 0033_cfop_ippt_write_mode exists ONLY on feature/cfop-ippt-write, never merged to
|
||||
master - this migration chains from 0032 (master's real head), not from 0033.
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0034_reference_data_registry"
|
||||
down_revision = "0032_contribuinte_ipi"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("datasets", sa.Column("licence_terms", sa.Text(), nullable=True))
|
||||
op.add_column("datasets", sa.Column("valid_from", sa.String(32), nullable=True))
|
||||
op.add_column("datasets", sa.Column("valid_to", sa.String(32), nullable=True))
|
||||
op.add_column("datasets", sa.Column("file_size_bytes", sa.Integer(), nullable=True))
|
||||
op.add_column("datasets", sa.Column("raw_file_path", sa.Text(), nullable=True))
|
||||
op.add_column("datasets", sa.Column("parse_errors", sa.JSON(), nullable=False, server_default="{}"))
|
||||
op.add_column("datasets", sa.Column("submitted_by", sa.String(128), nullable=True))
|
||||
op.add_column("datasets", sa.Column("approved_by", sa.String(128), nullable=True))
|
||||
op.add_column("datasets", sa.Column("approved_at", sa.DateTime(timezone=True), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
for name in ("approved_at", "approved_by", "submitted_by", "parse_errors", "raw_file_path", "file_size_bytes", "valid_to", "valid_from", "licence_terms"):
|
||||
op.drop_column("datasets", name)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""CFOP/IPPT write capability: per-installation write mode (DEFAULT OFF) + append-only audit table.
|
||||
|
||||
Revision ID: 0035_cfop_ippt_write_mode
|
||||
Revises: 0034_reference_data_registry
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0035_cfop_ippt_write_mode"
|
||||
down_revision = "0034_reference_data_registry"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("installations", sa.Column("cfop_ippt_write_mode", sa.String(24), nullable=False, server_default="DISABLED"))
|
||||
op.create_table(
|
||||
"cfop_ippt_write_audits",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False, index=True),
|
||||
sa.Column("installation_id", sa.Integer(), sa.ForeignKey("installations.id"), nullable=False, index=True),
|
||||
sa.Column("agent_id", sa.Integer(), sa.ForeignKey("agents.id"), nullable=False, index=True),
|
||||
sa.Column("job_id", sa.Integer(), nullable=True, index=True),
|
||||
sa.Column("product_code", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("field", sa.String(16), nullable=False),
|
||||
sa.Column("outcome", sa.String(16), nullable=False, index=True),
|
||||
sa.Column("reason", sa.String(96), nullable=True),
|
||||
sa.Column("old_value", sa.String(16), nullable=True),
|
||||
sa.Column("new_value", sa.String(16), nullable=True),
|
||||
sa.Column("action", sa.String(16), nullable=True),
|
||||
sa.Column("rollback_of", sa.Integer(), nullable=True, index=True),
|
||||
sa.Column("written_at", sa.String(40), nullable=True),
|
||||
sa.Column("recorded_at", sa.DateTime(timezone=True), server_default=sa.func.now(), index=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("cfop_ippt_write_audits")
|
||||
op.drop_column("installations", "cfop_ippt_write_mode")
|
||||
@@ -0,0 +1 @@
|
||||
"""Fiscal Hub application modules."""
|
||||
@@ -0,0 +1,256 @@
|
||||
"""Public, pre-authentication routes for Agent distribution.
|
||||
|
||||
GET /dl/agent/{token} and POST /v1/agents/enroll are reachable by anyone —
|
||||
same trust level as GET /login. Every other Agent-distribution route
|
||||
(create pending installation, generate a link, direct authenticated
|
||||
download, list/revoke releases) is a control-panel page and lives in
|
||||
web.py, gated the same way as the rest of the panel (admin session + CSRF).
|
||||
|
||||
The enrollment token embedded in a downloaded bundle is generated fresh at
|
||||
download time and its plaintext is never written to storage — only its
|
||||
SHA-256 hash persists, exactly like the download token that gated getting
|
||||
it. Neither is ever the Agent's own permanent Bearer token, which is
|
||||
minted only once, inside a successful POST /v1/agents/enroll.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import Response
|
||||
from sqlalchemy import select
|
||||
|
||||
from . import config, enrollment, release_storage
|
||||
from .audit import record_audit
|
||||
from .auth import token_hash as agent_token_hash
|
||||
from .models import Agent, AgentEnrollment, AgentRelease, Installation
|
||||
from .rate_limit import blocked, record_attempt
|
||||
|
||||
|
||||
def client_ip(request: Request) -> str:
|
||||
return request.client.host if request.client else "unknown"
|
||||
|
||||
|
||||
def _aware(value: datetime | None) -> datetime | None:
|
||||
"""SQLite (used by the test suite) drops tzinfo on round-trip even for a
|
||||
DateTime(timezone=True) column; Postgres never does. Normalize either way
|
||||
before comparing against datetime.now(timezone.utc) — same pattern as
|
||||
human_auth.py's session-expiry check."""
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def build_and_serve(session, enr: AgentEnrollment, request: Request, *, actor_admin_id: int | None = None) -> Response:
|
||||
"""Serve the release artifact for the installation — always the same
|
||||
universal release binary (ZIP bundle for legacy token flow, bare EXE for
|
||||
the installer flow). Audit the download; never embed the enrollment token
|
||||
in an EXE download (the operator types a standalone install code instead)."""
|
||||
release = session.get(AgentRelease, enr.release_id)
|
||||
if release is None or release.status != "PUBLISHED":
|
||||
raise enrollment.EnrollmentRejected("RELEASE_UNAVAILABLE")
|
||||
path = release_storage.safe_path_for(release.storage_path)
|
||||
if not path.exists():
|
||||
raise enrollment.EnrollmentRejected("RELEASE_FILE_MISSING")
|
||||
now = datetime.now(timezone.utc)
|
||||
is_exe = release.filename.lower().endswith(".exe")
|
||||
if is_exe:
|
||||
# Universal EXE flow: no per-installation bundle, no enrollment.json.
|
||||
# The operator types the standalone install code into Setup.exe and the
|
||||
# Hub resolves the installation from the code hash alone.
|
||||
enr.downloaded_at = now
|
||||
if enr.status not in ("ENROLLED", "REVOKED"):
|
||||
enr.status = "DOWNLOADED"
|
||||
record_audit(session, admin_user_id=actor_admin_id, tenant_id=enr.tenant_id, action="INSTALLER_DOWNLOADED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
|
||||
context={"ip": client_ip(request), "release_version": release.version, "release_id": release.id})
|
||||
session.commit()
|
||||
filename = release.filename
|
||||
return Response(content=path.read_bytes(), media_type="application/octet-stream",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"',
|
||||
"X-Content-Type-Options": "nosniff", "Cache-Control": "no-store"})
|
||||
token = enrollment.new_token()
|
||||
enr.enrollment_token_hash = enrollment.token_digest(token)
|
||||
enr.enrollment_token_expires_at = now + timedelta(seconds=config.settings.agent_enrollment_ttl_seconds)
|
||||
enr.enrollment_used_at = None
|
||||
enr.downloaded_at = now
|
||||
if enr.status not in ("ENROLLED", "REVOKED"):
|
||||
enr.status = "DOWNLOADED"
|
||||
plan = enrollment.BundlePlan(hub_url=config.settings.base_url, enrollment_uuid=enr.enrollment_uuid, enrollment_token=token)
|
||||
data = enrollment.build_bundle(path.read_bytes(), plan)
|
||||
record_audit(session, admin_user_id=actor_admin_id, tenant_id=enr.tenant_id, action="PACKAGE_DOWNLOADED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
|
||||
context={"ip": client_ip(request), "release_version": release.version, "release_id": release.id})
|
||||
session.commit()
|
||||
filename = f"GTIFiscalAgent-{release.version}-{enr.enrollment_uuid[:8]}.zip"
|
||||
return Response(content=data, media_type="application/octet-stream",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"',
|
||||
"X-Content-Type-Options": "nosniff", "Cache-Control": "no-store"})
|
||||
|
||||
|
||||
def router(session_factory):
|
||||
routes = APIRouter()
|
||||
|
||||
@routes.get("/dl/agent/{token}")
|
||||
def download_by_link(token: str, request: Request):
|
||||
ip = client_ip(request)
|
||||
if blocked("download", ip):
|
||||
raise HTTPException(429, "too many attempts")
|
||||
with session_factory() as session:
|
||||
candidates = session.scalars(select(AgentEnrollment).where(AgentEnrollment.download_token_hash.is_not(None))).all()
|
||||
enr = next((c for c in candidates if enrollment.token_matches(token, c.download_token_hash)), None)
|
||||
now = datetime.now(timezone.utc)
|
||||
try:
|
||||
if enr is None:
|
||||
raise enrollment.EnrollmentRejected("INVALID_LINK")
|
||||
if enr.status == "REVOKED":
|
||||
raise enrollment.EnrollmentRejected("REVOKED")
|
||||
if not enr.download_token_expires_at or _aware(enr.download_token_expires_at) < now:
|
||||
raise enrollment.EnrollmentRejected("EXPIRED_LINK")
|
||||
if enr.download_uses >= enr.download_max_uses:
|
||||
raise enrollment.EnrollmentRejected("LINK_ALREADY_USED")
|
||||
enr.download_uses += 1
|
||||
if enr.download_uses >= enr.download_max_uses:
|
||||
enr.download_token_hash = None # the link's last remaining use is spent: it dies immediately
|
||||
return build_and_serve(session, enr, request)
|
||||
except enrollment.EnrollmentRejected as rejected:
|
||||
session.rollback()
|
||||
record_attempt("download", ip)
|
||||
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id if enr else None, action="DOWNLOAD_LINK_REJECTED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid if enr else "-",
|
||||
context={"ip": ip, "reason": rejected.reason}, severity="WARNING")
|
||||
session.commit()
|
||||
raise HTTPException(404, "invalid or expired link")
|
||||
|
||||
@routes.post("/v1/agents/enroll")
|
||||
async def enroll(request: Request):
|
||||
ip = client_ip(request)
|
||||
if blocked("enroll", ip):
|
||||
raise HTTPException(429, "too many attempts")
|
||||
try:
|
||||
payload = await request.json()
|
||||
except Exception:
|
||||
payload = {}
|
||||
installation_id = str(payload.get("installation_id") or "")
|
||||
supplied_token = payload.get("enrollment_token")
|
||||
install_code = payload.get("install_code")
|
||||
# Adjustment: install_code and enrollment_token are mutually exclusive —
|
||||
# the EXE flow sends a code only; sending both is a request error.
|
||||
if install_code and (supplied_token or payload.get("installation_id")):
|
||||
record_attempt("enroll", ip)
|
||||
with session_factory() as session:
|
||||
record_audit(session, admin_user_id=None, tenant_id=None, action="ENROLLMENT_REJECTED",
|
||||
target_type="agent_enrollment", target_id="-",
|
||||
context={"ip": ip, "reason": "MIXED_CREDENTIALS"}, severity="WARNING")
|
||||
session.commit()
|
||||
raise HTTPException(400, "MIXED_CREDENTIALS")
|
||||
with session_factory() as session:
|
||||
enr = None
|
||||
try:
|
||||
if install_code:
|
||||
enr = _resolve_by_install_code(session, ip, install_code)
|
||||
else:
|
||||
enr = _resolve_by_legacy_token(session, ip, installation_id, supplied_token)
|
||||
if enr is None:
|
||||
raise enrollment.EnrollmentRejected("INVALID_INSTALLATION")
|
||||
if enr.status == "REVOKED":
|
||||
raise enrollment.EnrollmentRejected("REVOKED")
|
||||
if enr.status == "ENROLLED":
|
||||
raise enrollment.EnrollmentRejected("ALREADY_ENROLLED")
|
||||
release = session.get(AgentRelease, enr.release_id)
|
||||
if release is None or release.status != "PUBLISHED":
|
||||
raise enrollment.EnrollmentRejected("RELEASE_UNAVAILABLE")
|
||||
|
||||
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id, action="ENROLLMENT_STARTED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
|
||||
context={"ip": ip, "credential_type": "install_code" if install_code else "enrollment_token"})
|
||||
|
||||
hostname = str(payload.get("hostname") or "")[:255]
|
||||
mismatch = bool(enr.expected_hostname) and bool(hostname) and hostname.lower() != enr.expected_hostname.lower()
|
||||
|
||||
plain_token = secrets.token_urlsafe(32)
|
||||
agent = Agent(tenant_id=enr.tenant_id, name=enr.display_name, token_hash=agent_token_hash(plain_token),
|
||||
enabled=True, agent_version=str(payload.get("agent_version") or "")[:32] or None)
|
||||
session.add(agent)
|
||||
session.flush()
|
||||
session.add(Installation(agent_id=agent.id, tenant_id=enr.tenant_id, installation_id=enr.enrollment_uuid,
|
||||
display_name=enr.display_name, hostname=hostname or None,
|
||||
features_configured=bool(enr.features_selected), corrector_enabled=bool(enr.features_selected and enr.feature_corrector),
|
||||
enrichment_mode=(enr.feature_enrichment_mode if enr.features_selected else "DISABLED")))
|
||||
|
||||
enr.status = "ENROLLED"
|
||||
enr.enrollment_used_at = datetime.now(timezone.utc)
|
||||
enr.enrollment_token_hash = None # replay-proof even inside its TTL window
|
||||
enr.install_code_hash = None # consumed: truly one-time, replay-proof
|
||||
enr.install_code_expires_at = None
|
||||
enr.hostname_seen = hostname or None
|
||||
enr.machine_fingerprint_seen = str(payload.get("machine_fingerprint") or "")[:128] or None
|
||||
enr.hostname_mismatch = mismatch
|
||||
enr.agent_id = agent.id
|
||||
|
||||
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id, action="ENROLLMENT_SUCCEEDED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
|
||||
context={"ip": ip, "hostname": hostname, "hostname_mismatch": mismatch, "agent_id": agent.id})
|
||||
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id, action="AGENT_CREDENTIAL_ISSUED",
|
||||
target_type="agent", target_id=str(agent.id), context={"ip": ip, "installation": enr.enrollment_uuid})
|
||||
session.commit()
|
||||
return {"agent_id": agent.id, "agent_token": plain_token, "tenant_id": enr.tenant_id,
|
||||
"installation_id": enr.enrollment_uuid,
|
||||
"hub_url": config.settings.base_url, "hostname_mismatch": mismatch}
|
||||
except enrollment.EnrollmentRejected as rejected:
|
||||
session.rollback()
|
||||
record_attempt("enroll", ip)
|
||||
reason = rejected.reason
|
||||
if enr is not None:
|
||||
enr.install_code_attempts = (enr.install_code_attempts or 0) + 1
|
||||
record_attempt(f"enroll_code:{enr.enrollment_uuid}", ip)
|
||||
session.commit()
|
||||
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id if enr else None, action="ENROLLMENT_REJECTED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid if enr else (installation_id or "-"),
|
||||
context={"ip": ip, "reason": reason}, severity="WARNING")
|
||||
session.commit()
|
||||
raise HTTPException(401, reason)
|
||||
|
||||
def _resolve_by_install_code(session, ip, code):
|
||||
"""Resolve the Installation by the install_code hash alone. Returns the
|
||||
enrollment or None. Format/unknown codes and expired codes raise
|
||||
EnrollmentRejected. Per-installation and per-IP rate limits apply."""
|
||||
try:
|
||||
digest = enrollment.install_code_digest(code)
|
||||
except ValueError:
|
||||
raise enrollment.EnrollmentRejected("INVALID_INSTALL_CODE")
|
||||
enr = session.scalar(select(AgentEnrollment).where(AgentEnrollment.install_code_hash == digest))
|
||||
if enr is None:
|
||||
raise enrollment.EnrollmentRejected("INVALID_INSTALL_CODE")
|
||||
# Per-installation rate limit: a hostile caller guessing codes for one
|
||||
# installation must burn the whole budget even after the code rotates.
|
||||
if blocked(f"enroll_code:{enr.enrollment_uuid}", ip):
|
||||
raise enrollment.EnrollmentRejected("RATE_LIMITED")
|
||||
now = datetime.now(timezone.utc)
|
||||
if not enr.install_code_expires_at or _aware(enr.install_code_expires_at) < now:
|
||||
raise enrollment.EnrollmentRejected("EXPIRED")
|
||||
return enr
|
||||
|
||||
def _resolve_by_legacy_token(session, ip, installation_id, supplied_token):
|
||||
"""Legacy enrollment_token path (published ZIP bundles). Returns the
|
||||
enrollment or raises EnrollmentRejected. Order preserved from the
|
||||
original flow: enrolled-again must read ALREADY_ENROLLED, not
|
||||
INVALID_TOKEN."""
|
||||
if not enrollment.valid_uuid(installation_id):
|
||||
raise enrollment.EnrollmentRejected("INVALID_INSTALLATION")
|
||||
enr = session.scalar(select(AgentEnrollment).where(AgentEnrollment.enrollment_uuid == installation_id))
|
||||
if enr is None:
|
||||
raise enrollment.EnrollmentRejected("INVALID_INSTALLATION")
|
||||
if enr.status == "REVOKED":
|
||||
raise enrollment.EnrollmentRejected("REVOKED")
|
||||
if enr.status == "ENROLLED":
|
||||
raise enrollment.EnrollmentRejected("ALREADY_ENROLLED")
|
||||
now = datetime.now(timezone.utc)
|
||||
if not enr.enrollment_token_expires_at or _aware(enr.enrollment_token_expires_at) < now:
|
||||
raise enrollment.EnrollmentRejected("EXPIRED")
|
||||
if not enrollment.token_matches(supplied_token, enr.enrollment_token_hash):
|
||||
raise enrollment.EnrollmentRejected("INVALID_TOKEN")
|
||||
return enr
|
||||
|
||||
return routes
|
||||
@@ -0,0 +1,8 @@
|
||||
"""AI semantic layer V1 — NOT a fiscal authority.
|
||||
|
||||
Rules frozen by the operator contract:
|
||||
- AI only selects among official candidates; it never invents codes.
|
||||
- Cache first: KB exact/fingerprint hits never trigger AI calls.
|
||||
- No AI for origin, ST retention, or company-level decisions.
|
||||
- AI_WRITE_ALLOWED=false in V1: results stay AI_CANDIDATE_VALIDATED.
|
||||
"""
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Offline-safe, ID-correlated provider batch execution."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .provider import ProviderTransportError
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchRunResult:
|
||||
results_by_work_item_id: dict[str, dict] = field(default_factory=dict)
|
||||
errors: dict[str, str] = field(default_factory=dict)
|
||||
provider_calls: int = 0
|
||||
retries: int = 0
|
||||
usage: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
def run_batches(items, provider, batch_size: int, max_retries: int = 2, *, scenario: str | None = None) -> BatchRunResult:
|
||||
if batch_size <= 0:
|
||||
raise ValueError("batch_size must be positive")
|
||||
if max_retries < 0:
|
||||
raise ValueError("max_retries must not be negative")
|
||||
|
||||
result = BatchRunResult()
|
||||
for start in range(0, len(items), batch_size):
|
||||
batch = list(items[start:start + batch_size])
|
||||
expected_ids = [str(item.work_item_id) for item in batch]
|
||||
answers = None
|
||||
for attempt in range(max_retries + 1):
|
||||
result.provider_calls += 1
|
||||
try:
|
||||
kwargs = {} if scenario is None else {"scenario": scenario}
|
||||
answers = provider.select_batch(batch, **kwargs)
|
||||
break
|
||||
except ProviderTransportError as error:
|
||||
if attempt == max_retries:
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = str(error)
|
||||
else:
|
||||
result.retries += 1
|
||||
except Exception as error:
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = str(error)
|
||||
break
|
||||
if answers is None:
|
||||
continue
|
||||
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
|
||||
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
|
||||
error = "provider response did not contain exactly one answer per work_item_id"
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = error
|
||||
continue
|
||||
result.results_by_work_item_id.update({str(answer["work_item_id"]): answer for answer in answers})
|
||||
|
||||
result.usage = dict(getattr(provider, "usage", {}) or {})
|
||||
return result
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Persistent-boundary-compatible budget guard; provider invocation happens after this check."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from threading import Lock
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BudgetLimits:
|
||||
max_run_cost_usd: float = 0.54
|
||||
max_daily_cost_usd: float = 1.00
|
||||
max_monthly_cost_usd: float = 10.00
|
||||
max_requests_per_day: int = 100
|
||||
max_new_products_per_run: int = 750
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BudgetDecision:
|
||||
allowed: bool
|
||||
reason: str
|
||||
reserved_cost_usd: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UsageEntry:
|
||||
run_id: str
|
||||
tenant: str
|
||||
cost_usd: float
|
||||
requests: int
|
||||
new_products: int
|
||||
timestamp: str
|
||||
|
||||
|
||||
class BudgetLedger:
|
||||
"""Atomic in-process ledger with an explicit persistence boundary for Hub storage."""
|
||||
|
||||
def __init__(self, limits: BudgetLimits):
|
||||
self.limits = limits
|
||||
self.entries: list[UsageEntry] = []
|
||||
self._lock = Lock()
|
||||
|
||||
def check_and_reserve(self, run_id: str, tenant: str, cost_usd: float, requests: int, new_products: int) -> BudgetDecision:
|
||||
with self._lock:
|
||||
run_cost = sum(e.cost_usd for e in self.entries if e.run_id == run_id)
|
||||
day = datetime.now(timezone.utc).date()
|
||||
day_entries = [e for e in self.entries if e.timestamp[:10] == day.isoformat()]
|
||||
month_entries = [e for e in self.entries if e.timestamp[:7] == day.strftime("%Y-%m")]
|
||||
if run_cost + cost_usd > self.limits.max_run_cost_usd:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.cost_usd for e in day_entries) + cost_usd > self.limits.max_daily_cost_usd:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.cost_usd for e in month_entries) + cost_usd > self.limits.max_monthly_cost_usd:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.requests for e in day_entries) + requests > self.limits.max_requests_per_day:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.new_products for e in self.entries if e.run_id == run_id) + new_products > self.limits.max_new_products_per_run:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
self.entries.append(UsageEntry(run_id, tenant, cost_usd, requests, new_products, datetime.now(timezone.utc).isoformat()))
|
||||
return BudgetDecision(True, "RESERVED", cost_usd)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Pure cache and dataset-freshness policies for offline and real evaluations."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
VALID, NEEDS_REVALIDATION, CONFLICT = "VALID", "NEEDS_REVALIDATION", "CONFLICT"
|
||||
|
||||
|
||||
def cache_key(input_fingerprint, candidate_set_hash, model, prompt_version, dataset_versions):
|
||||
"""Return a stable key covering every input that can change an answer."""
|
||||
payload = {
|
||||
"input_fingerprint": str(input_fingerprint),
|
||||
"candidate_set_hash": str(candidate_set_hash),
|
||||
"model": str(model),
|
||||
"prompt_version": str(prompt_version),
|
||||
"dataset_versions": dict(sorted((dataset_versions or {}).items())),
|
||||
}
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def classify_knowledge_freshness(record_versions, active_versions):
|
||||
"""Classify versioned knowledge without deleting or rewriting history."""
|
||||
recorded = dict(record_versions or {})
|
||||
active = dict(active_versions or {})
|
||||
if any(key in recorded and key in active and recorded[key] != active[key] for key in recorded):
|
||||
return CONFLICT
|
||||
if recorded != active:
|
||||
return NEEDS_REVALIDATION
|
||||
return VALID
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Deterministic candidate generation from the ACTIVE official datasets.
|
||||
|
||||
The AI never receives an open question: for every task type the Hub first builds
|
||||
the official candidate list (NCM top-K by description, CEST links for the NCM,
|
||||
TIPI EX rows for the NCM). Providers may only pick from this list.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import CestEntry, CestNcmEntry, Dataset, NcmEntry, TipiEntry
|
||||
from .fingerprint import token_stems
|
||||
|
||||
|
||||
def _active_dataset(session: Session, kind: str):
|
||||
return session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
|
||||
|
||||
|
||||
def ncm_top_k(session: Session, description, k: int = 10):
|
||||
"""Top-K official NCM codes by (stemmed) token overlap with the product description."""
|
||||
dataset = _active_dataset(session, "NCM")
|
||||
if dataset is None:
|
||||
return []
|
||||
stems = token_stems(description)
|
||||
if not stems:
|
||||
return []
|
||||
scored = []
|
||||
for entry in session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all():
|
||||
score = len(stems & token_stems(entry.description)) / len(stems)
|
||||
if score > 0:
|
||||
scored.append((entry.code, entry.description, round(score, 4)))
|
||||
scored.sort(key=lambda row: (-row[2], row[0]))
|
||||
return scored[:k]
|
||||
|
||||
|
||||
def cest_candidates_for_ncm(session: Session, ncm: str):
|
||||
dataset = _active_dataset(session, "CEST")
|
||||
if dataset is None:
|
||||
return []
|
||||
rows = session.execute(
|
||||
select(CestEntry, CestNcmEntry.ncm_code)
|
||||
.join(CestNcmEntry, CestNcmEntry.cest_entry_id == CestEntry.id)
|
||||
.where(CestEntry.dataset_id == dataset.id)
|
||||
).all()
|
||||
return list(dict.fromkeys(entry for entry, ncm_code in rows if ncm.startswith(ncm_code)))
|
||||
|
||||
|
||||
def ex_tipi_candidates_for_ncm(session: Session, ncm: str):
|
||||
dataset = _active_dataset(session, "TIPI_IPI")
|
||||
if dataset is None:
|
||||
return []
|
||||
return list(dict.fromkeys(
|
||||
entry for entry in session.scalars(select(TipiEntry).where(TipiEntry.dataset_id == dataset.id, TipiEntry.ncm == ncm, TipiEntry.ex.is_not(None))).all()
|
||||
))
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Metrics and error attribution for the real LLM NCM evaluation (sections 8-9)."""
|
||||
|
||||
|
||||
def metrics(rows) -> dict:
|
||||
"""rows: [{gold_in_candidates, selected, gold, confidence, invalid}]"""
|
||||
total = len(rows)
|
||||
correct = sum(1 for row in rows if not row["invalid"] and row["selected"] == row["gold"])
|
||||
wrong = sum(1 for row in rows if not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"])
|
||||
abstain = sum(1 for row in rows if not row["invalid"] and row["selected"] is None)
|
||||
invalid = sum(1 for row in rows if row["invalid"])
|
||||
gold_in = sum(1 for row in rows if row["gold_in_candidates"])
|
||||
answered = correct + wrong
|
||||
high_conf = [row for row in rows if not row["invalid"] and row["confidence"] >= 0.95 and row["selected"] is not None]
|
||||
high_conf_wrong = sum(1 for row in high_conf if row["selected"] != row["gold"])
|
||||
return {
|
||||
"TOTAL": total,
|
||||
"CORRECT": correct,
|
||||
"WRONG": wrong,
|
||||
"ABSTAIN": abstain,
|
||||
"INVALID": invalid,
|
||||
"GOLD_IN_CANDIDATES": gold_in,
|
||||
"ACCURACY_ALL": round(correct / total, 4) if total else None,
|
||||
"SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE": round(correct / answered, 4) if answered else None,
|
||||
"PRECISION_WHEN_HIGH_CONFIDENCE": round(sum(1 for row in high_conf if row["selected"] == row["gold"]) / len(high_conf), 4) if high_conf else None,
|
||||
"HIGH_CONFIDENCE_WRONG_RATE": round(high_conf_wrong / len(high_conf), 4) if high_conf else None,
|
||||
"ABSTENTION_RATE": round(abstain / total, 4) if total else None,
|
||||
}
|
||||
|
||||
|
||||
def attribution(rows) -> dict:
|
||||
return {
|
||||
"GOLD_NOT_IN_CANDIDATES": sum(1 for row in rows if not row["gold_in_candidates"]),
|
||||
"GOLD_IN_CANDIDATES_BUT_WRONG": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"]),
|
||||
"GOLD_IN_CANDIDATES_BUT_ABSTAIN": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is None),
|
||||
"INVALID_OUTPUT": sum(1 for row in rows if row["invalid"]),
|
||||
}
|
||||
|
||||
|
||||
_BUCKETS = ((">=0.99", 0.99, 1.01), ("0.95-0.9899", 0.95, 0.99), ("0.90-0.9499", 0.90, 0.95), ("0.80-0.8999", 0.80, 0.90), ("<0.80", -0.01, 0.80))
|
||||
|
||||
|
||||
def confidence_buckets(rows) -> dict:
|
||||
out = {name: {"total": 0, "correct": 0} for name, _, _ in _BUCKETS}
|
||||
for row in rows:
|
||||
if row["invalid"] or row["selected"] is None:
|
||||
continue
|
||||
for name, low, high in _BUCKETS:
|
||||
if low <= row["confidence"] < high:
|
||||
out[name]["total"] += 1
|
||||
if row["selected"] == row["gold"]:
|
||||
out[name]["correct"] += 1
|
||||
break
|
||||
return out
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Deterministic provider responses for offline pipeline tests."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from .llm_provider import LlmAnswerError, LlmEvalItem, validate_answer
|
||||
from .provider import ProviderContractError, ProviderTransportError
|
||||
|
||||
|
||||
SCENARIOS = frozenset({
|
||||
"CORRECT", "WRONG", "ABSTAIN", "INVALID_JSON", "MISSING_ITEM",
|
||||
"DUPLICATE_ITEM", "OUT_OF_CANDIDATE", "TIMEOUT", "429", "500",
|
||||
})
|
||||
|
||||
|
||||
class DeterministicFakeSemanticProvider:
|
||||
"""Return repeatable answers without persistence or network access."""
|
||||
|
||||
name = "deterministic-fake"
|
||||
model = "deterministic-fake-v1"
|
||||
prompt_version = "fake-provider-v1"
|
||||
|
||||
def __init__(self, *, shuffle: bool = False, fail_work_item_ids: Iterable[str] = (), failure_scenario: str = "TIMEOUT"):
|
||||
self.shuffle = shuffle
|
||||
self.fail_work_item_ids = {str(item_id) for item_id in fail_work_item_ids}
|
||||
self.failure_scenario = failure_scenario
|
||||
self.calls = 0
|
||||
self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
|
||||
|
||||
def select_batch(self, items: list[LlmEvalItem], scenario: str | None = None) -> list[dict]:
|
||||
scenario = scenario or "CORRECT"
|
||||
if scenario not in SCENARIOS:
|
||||
raise ValueError(f"unknown fake provider scenario: {scenario}")
|
||||
self.calls += 1
|
||||
if self.fail_work_item_ids and len(items) == 1 and str(items[0].work_item_id) in self.fail_work_item_ids:
|
||||
self._raise_transport(self.failure_scenario)
|
||||
if scenario in {"TIMEOUT", "429", "500"}:
|
||||
self._raise_transport(scenario)
|
||||
if scenario == "INVALID_JSON":
|
||||
raise LlmAnswerError("fake provider returned invalid JSON")
|
||||
|
||||
answers = [self._answer(item, scenario) for item in items]
|
||||
if scenario == "MISSING_ITEM" and answers:
|
||||
answers.pop()
|
||||
elif scenario == "DUPLICATE_ITEM" and answers:
|
||||
answers[-1] = dict(answers[0])
|
||||
if self.shuffle:
|
||||
answers.reverse()
|
||||
expected_ids = [str(item.work_item_id) for item in items]
|
||||
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
|
||||
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
|
||||
raise ProviderContractError("fake provider response did not contain exactly one answer per work_item_id")
|
||||
items_by_id = {str(item.work_item_id): item for item in items}
|
||||
try:
|
||||
return [validate_answer(answer, items_by_id[str(answer["work_item_id"])]) for answer in answers]
|
||||
except (LlmAnswerError, KeyError, TypeError) as error:
|
||||
raise ProviderContractError(str(error)) from error
|
||||
|
||||
@staticmethod
|
||||
def _answer(item: LlmEvalItem, scenario: str) -> dict:
|
||||
candidates = [str(candidate[0]) for candidate in item.candidates]
|
||||
selected = None
|
||||
abstain = scenario == "ABSTAIN"
|
||||
if not abstain and candidates:
|
||||
selected = "99999999" if scenario == "OUT_OF_CANDIDATE" else candidates[min(1, len(candidates) - 1) if scenario == "WRONG" else 0]
|
||||
return {
|
||||
"work_item_id": item.work_item_id,
|
||||
"selected_ncm": selected,
|
||||
"confidence": 0.0 if abstain else 0.75,
|
||||
"abstain": abstain or selected is None,
|
||||
"reason_short": f"fake scenario {scenario}",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _raise_transport(scenario: str) -> None:
|
||||
if scenario not in {"TIMEOUT", "429", "500"}:
|
||||
raise ValueError(f"unknown fake transport scenario: {scenario}")
|
||||
raise ProviderTransportError(f"fake provider transport failure: {scenario}")
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Product identity: stable fingerprints and GTIN validation.
|
||||
|
||||
Global identity uses ONLY stable attributes (normalized description, brand, unit,
|
||||
package quantity, GTIN). Fiscal/company context (CFOP, CSOSN, CST, rate, ST, tenant)
|
||||
never participates: those vary per context and must not merge distinct knowledge.
|
||||
"""
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from ..classification import ENGINE_VERSION # noqa: F401 (re-exported for the AI layer)
|
||||
|
||||
|
||||
def normalize_description(value) -> str:
|
||||
"""NFKD -> ascii -> A-Z0-9 tokens, space-joined (same canonical form as classification)."""
|
||||
text = unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper()
|
||||
return " ".join(re.sub(r"[^A-Z0-9]+", " ", text).split())
|
||||
|
||||
|
||||
def _normalize_token(value) -> str:
|
||||
return normalize_description(value)
|
||||
|
||||
|
||||
def _normalize_quantity(value) -> str:
|
||||
digits = re.sub(r"[^0-9.,]", "", str(value or ""))
|
||||
if not digits:
|
||||
return ""
|
||||
try:
|
||||
number = float(digits.replace(",", "."))
|
||||
except ValueError:
|
||||
return digits
|
||||
return f"{number:g}"
|
||||
|
||||
|
||||
def product_fingerprint(description, brand=None, unit=None, package_quantity=None) -> str:
|
||||
parts = [_normalize_token(description), _normalize_token(brand), _normalize_token(unit), _normalize_quantity(package_quantity)]
|
||||
return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _stem(token: str) -> str:
|
||||
"""Minimal Portuguese plural stemming so AGUAS matches AGUA / MINERAIS matches MINERAL."""
|
||||
if len(token) > 4 and token.endswith("AIS"):
|
||||
return token[:-3] + "AL"
|
||||
if len(token) > 4 and token.endswith("ES"):
|
||||
return token[:-2]
|
||||
if len(token) > 3 and token.endswith("S"):
|
||||
return token[:-1]
|
||||
return token
|
||||
|
||||
|
||||
def token_stems(value) -> set:
|
||||
return {_stem(token) for token in normalize_description(value).split()}
|
||||
|
||||
|
||||
def overlap_score(query, official) -> float:
|
||||
"""Fraction of query description stems present in the official description stems."""
|
||||
stems = token_stems(query)
|
||||
if not stems:
|
||||
return 0.0
|
||||
return len(stems & token_stems(official)) / len(stems)
|
||||
|
||||
|
||||
def validate_gtin(value) -> str | None:
|
||||
"""Returns the canonical GTIN when the checksum is valid, else None.
|
||||
Accepts GTIN-8/12/13/14 (EAN-8/UPC-12/EAN-13/GTIN-14)."""
|
||||
digits = re.sub(r"\D", "", str(value or ""))
|
||||
if len(digits) not in (8, 12, 13, 14):
|
||||
return None
|
||||
total = 0
|
||||
for index, char in enumerate(reversed(digits)):
|
||||
digit = int(char)
|
||||
total += digit * (1 if index % 2 == 0 else 3)
|
||||
if total % 10 != 0:
|
||||
return None
|
||||
return digits
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user