fix: deploy health via docker exec (prod port bound to VM IP), safe rollback, error propagation, VPN retry, release default + download verification

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

central/scripts/verify_release_download.py

@@ -0,0 +1,46 @@
+"""Post-deploy proof through the PUBLIC panel: the default release is served as the real installer EXE (not a zip)
+with the published sha256. Creates a throwaway LOCAL admin + an E2E installation, revokes/deactivates both after
+(never deleted: audit FKs). Run inside the Hub container: python central/scripts/verify_release_download.py <version>"""
+import hashlib
+import sys
+import time
+
+import httpx
+from sqlalchemy.orm import Session
+
+sys.path.insert(0, "central")
+from app.db import engine  # noqa: E402
+from app.human_auth import hash_password  # noqa: E402
+from app.models import AdminUser, AgentRelease, Tenant  # noqa: E402
+
+BASE = "https://gtifiscal.gtigrupo.com.br"
+PASSWORD = "Verify-tmp-2026!x"
+want = sys.argv[1]
+user = f"e2e-verify-{int(time.time())}"
+with Session(engine) as db:
+    rel = db.query(AgentRelease).filter_by(version=want, platform="windows", architecture="x64").one()
+    default = db.query(AgentRelease).filter_by(status="PUBLISHED").order_by(AgentRelease.created_at.desc()).first()
+    assert rel.status == "PUBLISHED" and default.id == rel.id, f"default release is {default.version}, expected {want}"
+    tenant = db.query(Tenant).filter_by(name="E2E-PROD-TEST").one_or_none() or Tenant(name="E2E-PROD-TEST")
+    db.add(tenant); db.add(AdminUser(username=user, password_hash=hash_password(PASSWORD), is_active=True, auth_source="LOCAL")); db.flush()
+    tenant_id, release_id, sha, name = tenant.id, rel.id, rel.sha256, rel.filename
+    db.commit()
+try:
+    c = httpx.Client(follow_redirects=False, timeout=120)
+    assert c.post(f"{BASE}/login", data={"username": user, "password": PASSWORD}).status_code == 303
+    csrf = c.cookies.get("gti_csrf")
+    r = c.post(f"{BASE}/agents/new", data={"csrf_token": csrf, "tenant_id": tenant_id, "display_name": "E2E-VERIFY Servidor", "release_id": release_id})
+    assert r.status_code == 303, r.status_code
+    uuid = r.headers["location"].rsplit("/", 1)[-1]
+    d = c.get(f"{BASE}/agents/pending/{uuid}/download")
+    disp = d.headers.get("content-disposition", "")
+    assert d.status_code == 200 and name in disp and name.endswith(".exe"), (d.status_code, disp)
+    assert d.content[:2] == b"MZ", "not a Windows executable"
+    got = hashlib.sha256(d.content).hexdigest()
+    assert got == sha, "sha256 mismatch"
+    print(f"DOWNLOAD_IS_EXE=true filename={name} SHA256_MATCH=true size={len(d.content)}")
+    c.post(f"{BASE}/agents/pending/{uuid}/revoke", data={"csrf_token": csrf})
+finally:
+    with Session(engine) as db:
+        db.query(AdminUser).filter_by(username=user).update({"is_active": False}); db.commit()
+print("VERIFY_DOWNLOAD=PASS")

scripts/deploy_hub.ps1

@@ -48,11 +48,29 @@ try {
         if ($LASTEXITCODE -ne 0) { throw "tests failed - deploy aborted" }
     }
 
+    # 4. route/SSH recovery: reconnect the existing Windows VPN profile "GTI" (saved credentials) and retry
+    $reachable = $false
+    for ($i = 1; $i -le 3 -and -not $reachable; $i++) {
+        $reachable = (Test-NetConnection 10.0.10.117 -Port 22 -WarningAction SilentlyContinue).TcpTestSucceeded
+        if (-not $reachable) {
+            Write-Host "SSH_UNREACHABLE attempt=$i - reconnecting VPN profile GTI"
+            rasdial GTI | Out-Null
+            Start-Sleep -Seconds (10 * $i)
+        }
+    }
+    if (-not $reachable) { throw "FAILED_STAGE=network REMOTE_ERROR=10.0.10.117:22 unreachable after 3 attempts (VPN profile GTI)" }
+    Write-Host "SSH_ROUTE_OK"
+    Remove-Item (Join-Path $LogDir "last_deploy_result.json") -ErrorAction SilentlyContinue
+
     $deployArgs = @("scripts\deploy_hub_remote.py", "--version", $Version)
     if ($DryRun) { $deployArgs += "--dry-run" }
     if ($PublishAgent) { $deployArgs += "--publish-agent" }
     & $Python @deployArgs
-    if ($LASTEXITCODE -ne 0) { throw "remote deploy failed (see ROLLBACK_REQUIRED above)" }
+    if ($LASTEXITCODE -ne 0) {
+        $res = Join-Path $LogDir "last_deploy_result.json"
+        $detail = if (Test-Path $res) { $j = Get-Content $res -Raw | ConvertFrom-Json; "FAILED_STAGE/ERROR=$($j.error) ROLLBACK_REQUIRED=$($j.rollback_required)" } else { "no result file (see $Log)" }
+        throw "remote deploy failed: $detail"
+    }
     Write-Host "DEPLOY_SCRIPT_OK log=$Log"
 } finally {
     Remove-Item Env:GTI_SSH_PASSWORD -ErrorAction SilentlyContinue

scripts/deploy_hub_remote.py

@@ -13,6 +13,7 @@ import re
 import subprocess
 import sys
 import time
+import urllib.error
 import urllib.request
 from pathlib import Path
 
@@ -75,8 +76,9 @@ def wait_health(r: Remote, expected: str) -> None:
         time.sleep(4)
     else:
         raise RuntimeError("containers did not become healthy")
-    ver = json.loads(r.run("curl -sf http://127.0.0.1:8000/version", mutating=False))
-    r.run("curl -sf -o /dev/null -w '%{http_code}' http://127.0.0.1:8000/health", mutating=False)
+    probe = ("python -c \"import urllib.request as u;print(u.urlopen('http://localhost:8000/%s').read().decode())\"")
+    ver = json.loads(r.run(f"docker exec {HUB} " + probe % "version", mutating=False))
+    r.run(f"docker exec {HUB} " + probe % "health", mutating=False)
     if ver.get("version") != expected:
         raise RuntimeError(f"version mismatch: production {ver.get('version')} != expected {expected}")
     log(f"HEALTH_OK version={ver.get('version')}")
@@ -93,6 +95,26 @@ def public_smoke(expected: str) -> None:
     log(f"PUBLIC_HEALTH=200 PUBLIC_VERSION={expected}")
 
 
+def write_result(status: str, error: str, rollback: bool) -> None:
+    out = REPO / "logs"
+    out.mkdir(exist_ok=True)
+    text = json.dumps({"status": status, "error": SECRET_RE.sub(r"
***", error).replace(PASSWORD, "***"), "rollback_required": rollback})
+    (out / "last_deploy_result.json").write_text(text, encoding="utf-8")
+
+
+def public_extra() -> None:
+    body = urllib.request.urlopen("https://gtigerenciador.gtigrupo.com.br/health", timeout=20).status
+    log(f"GERENCIADOR_HEALTH={body}")
+    req = urllib.request.Request(PUBLIC + "/auth/sso/callback", data=b"assertion=bogus", method="POST")
+    try:
+        code = urllib.request.urlopen(req, timeout=20).status
+    except urllib.error.HTTPError as e:
+        code = e.code
+    if code >= 500:
+        raise RuntimeError(f"SSO callback misbehaves: {code}")
+    log(f"SSO_SMOKE=ok (bogus assertion -> {code}, no 5xx)")
+
+
 def main() -> int:
     ap = argparse.ArgumentParser()
     ap.add_argument("--dry-run", action="store_true")
@@ -115,28 +137,41 @@ def main() -> int:
 
     r = Remote(a.dry_run)
     touched = False
+    st = {"stage": "connect"}
     try:
+        st["stage"] = "preflight"
         r.run("echo SSH_OK; docker ps --format '{{.Names}}' | grep -c gti-fiscal-hub", mutating=False)
+        free = r.run(f"df -Pm {REMOTE} | tail -1 | awk '{{print $4}}'", mutating=False).split()
+        if free and free[0].isdigit() and int(free[0]) < 2000:
+            raise RuntimeError(f"remote disk too low: {free[0]} MB free")
+        r.run(f"docker exec {HUB} python -c \"import urllib.request as u;print(u.urlopen('http://localhost:8000/version').read().decode())\"", check=False, mutating=False)
+        st["stage"] = "backup"
         # 4. backup: DB + release storage + env/config + previous tree. Failure aborts before anything changes.
         r.run(f"mkdir -p {bdir}")
         r.run(f"docker exec {PG} pg_dump -U gti -d gti_fiscal -Fc > {bdir}/gti_fiscal.dump && test -s {bdir}/gti_fiscal.dump")
         r.run(f"docker exec {HUB} tar -C /app/data -cz releases > {bdir}/releases.tgz && test -s {bdir}/releases.tgz")
         r.run(f"cp {COMPOSE_DIR}/.env {bdir}/hub.env && chmod 600 {bdir}/hub.env")
-        r.run(f"tar -czf {bdir}/previous_tree.tgz -C {REMOTE} . && test -s {bdir}/previous_tree.tgz")
+        r.run(f"tar -czf {bdir}/previous_tree.tgz -C {REMOTE} --exclude=./deploy/hub/backups . && test -s {bdir}/previous_tree.tgz")
         log(f"BACKUP_DIR={bdir}")
         # 5. transfer (git archive never contains .env / secrets; .env.local is gitignored)
+        st["stage"] = "upload"
         r.put(tar_local, "/tmp/gti_fiscal_deploy.tar")
         touched = True
+        st["stage"] = "extract"
         r.run(f"tar -xf /tmp/gti_fiscal_deploy.tar -C {REMOTE} && rm -f /tmp/gti_fiscal_deploy.tar")
         r.run(f"sed -i 's/^GTI_FISCAL_VERSION=.*/GTI_FISCAL_VERSION={version}/' {COMPOSE_DIR}/.env")
+        st["stage"] = "docker_build_up"
         # 6/7. only the Hub is rebuilt; Postgres and the Gerenciador stack are untouched. Migrations run at start.
         r.run(f"cd {COMPOSE_DIR} && {COMPOSE} build fiscal-hub && {COMPOSE} up -d fiscal-hub", timeout=900)
+        st["stage"] = "migration_head"
         head = r.run(f"sleep 10; docker exec {HUB} python -m alembic -c central/alembic.ini current", check=False, mutating=False)
         if not a.dry_run and "(head)" not in head:
             raise RuntimeError("alembic is not at head after start")
+        st["stage"] = "health"
         wait_health(r, version)
         # 9. Agent release (immutable: publish_agent_release refuses a different sha256 for an existing version)
         if a.publish_agent:
+            st["stage"] = "publish_release"
             digest = hashlib.sha256(installer.read_bytes()).hexdigest()
             r.put(installer, f"/tmp/{installer.name}")
             r.run(f"docker cp /tmp/{installer.name} {HUB}:/tmp/ && rm -f /tmp/{installer.name}")
@@ -145,19 +180,32 @@ def main() -> int:
                   f"print(engine.connect().execute(text(\\\"select version,status,size_bytes,sha256 from agent_releases where version='{version}'\\\")).fetchall())\"",
                   check=False)
             log(f"EXPECTED_SHA256={digest} SIZE={installer.stat().st_size}")
+            # older Setup installers stop being offered (never deleted); the newest PUBLISHED release is the default
+            r.run(f"docker exec {HUB} python -c \"from central.app.db import engine; from sqlalchemy import text; c=engine.begin().__enter__(); "
+                  f"print('revoked', c.execute(text(\\\"update agent_releases set status='REVOKED' where status='PUBLISHED' and filename like 'GTIFiscalAgent-Setup-%.exe' and version<>'{version}'\\\")).rowcount)\"",
+                  check=False)
+            st["stage"] = "verify_download"
+            r.run(f"docker exec {HUB} python central/scripts/verify_release_download.py {version}")
         else:
             log("RELEASE_PUBLISH=skipped")
         if not a.dry_run:
+            st["stage"] = "public_smoke"
             public_smoke(version)
+            public_extra()
+        st["stage"] = "done"
+        write_result("OK", "", False)
         log("ROLLBACK_REQUIRED=false")
         log("DEPLOY_OK" if not a.dry_run else "DRY_RUN_OK")
         return 0
     except Exception as exc:  # noqa: BLE001
+        log(f"FAILED_STAGE={st['stage']}")
+        log(f"REMOTE_ERROR={exc}")
         log(f"DEPLOY_FAILED: {exc}")
+        write_result("FAILED", f"{st['stage']}: {exc}", touched and not a.dry_run)
         if touched and not a.dry_run:
             log("ROLLBACK_REQUIRED=true — restoring previous tree (Postgres volume is NOT touched)")
             try:
-                r.run(f"tar -xzf {bdir}/previous_tree.tgz -C {REMOTE}")
+                r.run(f"tar -xzf {bdir}/previous_tree.tgz -C {REMOTE} --no-same-owner -m --exclude=./deploy/hub/backups")
                 r.run(f"cd {COMPOSE_DIR} && {COMPOSE} build fiscal-hub && {COMPOSE} up -d fiscal-hub", timeout=900)
                 log(f"ROLLBACK_APPLIED. DB dump kept at {bdir}/gti_fiscal.dump (restore manually only if a migration ran)")
             except Exception as rb:  # noqa: BLE001