fix(v2): AI_FALLBACK replaces the ACTIONABLE_NOW-with-confidence-0 lie, normalize null-like strings

Fallback (ai_disabled, ai_error, invalid_json, ai_max_candidates_cap --
any candidate the AI never actually analyzed) now produces classification
'AI_FALLBACK', an explicit technical state that is never part of the
11-class enum sent to OpenAI (the AI itself can never legitimately return
it). Previously it silently claimed ACTIONABLE_NOW with confidence 0,
which made unanalyzed tickets indistinguishable from genuinely confirmed
ones at the classification-label level.

Also normalizes blocker_type/waiting_for when the model returns the
literal string "null"/"none"/"n/a"/"" instead of JSON null -- done
once, in the validation layer (logic/classification.js), not scattered
around the workflow.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Gustavo
2026-08-24 16:49:22 -03:00
co-authored by Claude Sonnet 5
parent db7a261a10
commit 00a42db269
6 changed files with 87 additions and 11 deletions
+15 -3
View File
@@ -4,6 +4,18 @@ const VALID_CLASSIFICATIONS = [
'PROJECT_LONG_TERM', 'DUPLICATE_OR_OBSOLETE', 'INSUFFICIENT_CONTEXT',
];
// O modelo, apesar do schema permitir null de verdade em blocker_type/waiting_for, as vezes
// devolve a STRING "null" (ou variantes equivalentes) em vez do valor JSON null. Normaliza
// aqui, na camada de validacao -- unico lugar que toca a resposta crua da IA.
const NULL_LIKE_STRINGS = new Set(['null', 'none', 'n/a', 'na', '']);
function normalizeNullableString(value) {
if (value === null || value === undefined) return null;
const trimmed = String(value).trim();
if (NULL_LIKE_STRINGS.has(trimmed.toLowerCase())) return null;
return trimmed;
}
function validateAiClassification(raw) {
const errors = [];
if (!raw || typeof raw !== 'object') {
@@ -28,8 +40,8 @@ function validateAiClassification(raw) {
classification: raw.classification,
actionable_today: raw.actionable_today,
confidence,
blocker_type: raw.blocker_type ?? null,
waiting_for: raw.waiting_for ?? null,
blocker_type: normalizeNullableString(raw.blocker_type),
waiting_for: normalizeNullableString(raw.waiting_for),
next_action: raw.next_action,
suggested_action_type: raw.suggested_action_type ?? 'technical_action',
business_risk: raw.business_risk ?? 'medium',
@@ -41,4 +53,4 @@ function validateAiClassification(raw) {
};
}
module.exports = { VALID_CLASSIFICATIONS, validateAiClassification };
module.exports = { VALID_CLASSIFICATIONS, validateAiClassification, normalizeNullableString };
+23
View File
@@ -52,3 +52,26 @@ test('todas as classes do dominio sao aceitas', () => {
assert.equal(result.valid, true, `classe ${classification} deveria ser valida`);
}
});
test('AI_FALLBACK nao esta no dominio de classes que a IA pode retornar', () => {
assert.ok(!VALID_CLASSIFICATIONS.includes('AI_FALLBACK'));
});
test('blocker_type/waiting_for como string "null" sao normalizados para null de verdade', () => {
const result = validateAiClassification(validRaw({ blocker_type: 'null', waiting_for: 'None' }));
assert.equal(result.valid, true);
assert.equal(result.data.blocker_type, null);
assert.equal(result.data.waiting_for, null);
});
test('blocker_type/waiting_for como "n/a" ou string vazia tambem viram null', () => {
const result = validateAiClassification(validRaw({ blocker_type: 'N/A', waiting_for: '' }));
assert.equal(result.data.blocker_type, null);
assert.equal(result.data.waiting_for, null);
});
test('blocker_type/waiting_for com valor real nao sao afetados pela normalizacao', () => {
const result = validateAiClassification(validRaw({ blocker_type: 'fornecedor', waiting_for: 'Patricia' }));
assert.equal(result.data.blocker_type, 'fornecedor');
assert.equal(result.data.waiting_for, 'Patricia');
});
+11 -4
View File
@@ -1,11 +1,18 @@
// AI_FALLBACK e um estado tecnico: significa "a IA nunca analisou este chamado" (erro,
// timeout, IA desligada, JSON invalido, ou cortado pelo limite de AI_MAX_CANDIDATES). NAO e
// uma conclusao da IA -- por isso nunca usa uma das 11 classes que a IA pode retornar
// (ver logic/classification.js). Politica de elegibilidade fica centralizada em
// logic/cooldown.js (applyFallbackPolicy), nao aqui.
const AI_FALLBACK_CLASSIFICATION = 'AI_FALLBACK';
function buildFallbackClassification(reason) {
return {
classification: 'ACTIONABLE_NOW',
actionable_today: true,
classification: AI_FALLBACK_CLASSIFICATION,
actionable_today: false,
confidence: 0,
blocker_type: null,
waiting_for: null,
next_action: 'Revisar chamado manualmente',
next_action: 'Revisar chamado manualmente (nao analisado pela IA)',
suggested_action_type: 'manual_review',
business_risk: 'medium',
reason: `Fallback deterministico: ${reason}`,
@@ -15,4 +22,4 @@ function buildFallbackClassification(reason) {
};
}
module.exports = { buildFallbackClassification };
module.exports = { buildFallbackClassification, AI_FALLBACK_CLASSIFICATION };
+31 -2
View File
@@ -1,12 +1,41 @@
const { test } = require('node:test');
const assert = require('node:assert/strict');
const { buildFallbackClassification } = require('./fallback');
const { buildFallbackClassification, AI_FALLBACK_CLASSIFICATION } = require('./fallback');
const { VALID_CLASSIFICATIONS } = require('./classification');
test('fallback e sempre seguro e marcado como tal', () => {
const result = buildFallbackClassification('ai_timeout');
assert.equal(result.is_fallback, true);
assert.equal(result.fallback_reason, 'ai_timeout');
assert.equal(result.confidence, 0);
assert.equal(result.classification, 'ACTIONABLE_NOW');
assert.ok(result.reason.includes('Fallback'));
});
test('fallback NUNCA finge ser ACTIONABLE_NOW -- usa AI_FALLBACK, um estado tecnico explicito', () => {
const result = buildFallbackClassification('ai_error');
assert.equal(result.classification, AI_FALLBACK_CLASSIFICATION);
assert.notEqual(result.classification, 'ACTIONABLE_NOW');
assert.equal(result.actionable_today, false);
});
test('AI_FALLBACK nunca e uma classe que a propria IA pode retornar (fica fora do schema enviado a OpenAI)', () => {
assert.ok(!VALID_CLASSIFICATIONS.includes(AI_FALLBACK_CLASSIFICATION));
});
test('AI_MAX_CANDIDATES gera AI_FALLBACK, nao ACTIONABLE_NOW', () => {
const result = buildFallbackClassification('ai_max_candidates_cap');
assert.equal(result.classification, AI_FALLBACK_CLASSIFICATION);
assert.equal(result.fallback_reason, 'ai_max_candidates_cap');
});
test('erro de IA gera AI_FALLBACK', () => {
assert.equal(buildFallbackClassification('ai_error').classification, AI_FALLBACK_CLASSIFICATION);
});
test('IA desligada gera AI_FALLBACK', () => {
assert.equal(buildFallbackClassification('ai_disabled').classification, AI_FALLBACK_CLASSIFICATION);
});
test('JSON invalido gera AI_FALLBACK', () => {
assert.equal(buildFallbackClassification('invalid_json').classification, AI_FALLBACK_CLASSIFICATION);
});
@@ -17,3 +17,8 @@ test('SQL de candidatos V2 NAO tem mais LIMIT fixo em 10 (funil maior)', () => {
assert.ok(sql.includes('LIMIT {CANDIDATE_POOL_SIZE}'));
assert.ok(!sql.includes('LIMIT 10'));
});
test('SQL de candidatos V2 traz a descricao real do chamado (t.content) para a IA', () => {
const sql = fs.readFileSync(path.join(__dirname, '..', '..', 'sql', 'candidatos_v2_glpi.sql'), 'utf8');
assert.ok(sql.includes('t.content AS descricao'));
});
+2 -2
View File
@@ -36,7 +36,7 @@ WITH ultima_interacao AS (
GROUP BY tu.tickets_id
), chamados_base AS (
SELECT
t.id, t.name AS titulo, t.status,
t.id, t.name AS titulo, t.content AS descricao, t.status,
CASE t.status WHEN 1 THEN 'Novo' WHEN 2 THEN 'Em atendimento' WHEN 3 THEN 'Planejado' WHEN 4 THEN 'Pendente' ELSE CONCAT('Status ', t.status) END AS status_nome,
t.date AS aberto_em, t.date_mod AS modificado_em, t.time_to_resolve AS sla_resolver_ate,
t.priority, t.urgency, t.impact,
@@ -63,7 +63,7 @@ WITH ultima_interacao AS (
AND COALESCE(c.completename, c.name, '') NOT LIKE 'INFRA%'
), chamados_ranked AS (
SELECT
id, titulo, status, status_nome, aberto_em, modificado_em, ultima_interacao_em,
id, titulo, descricao, status, status_nome, aberto_em, modificado_em, ultima_interacao_em,
dias_aberto, dias_sem_interacao, sla_resolver_ate, horas_ate_sla, sla_vencido, sla_em_risco,
priority AS prioridade, urgency AS urgencia, impact AS impacto,
entidade, categoria, tecnico_atribuido, sem_tecnico, link_glpi,