From e9f585458f0ff61062969ade6cd76ae7e8c751bd Mon Sep 17 00:00:00 2001 From: Gustavo Date: Fri, 31 Jul 2026 11:52:48 -0300 Subject: [PATCH] Fix perf flakiness in shared-number candidate query by precomputing lead numbers once The candidate query's lead_number/lead_equivalent forms were being re-embedded as raw SQL expression text at every usage site (verification join, opt-out check, shared-number check, inbound-replies check). Each embedding re-evaluates the same deeply nested replace()/substr()/length() chain per row; adding the shared-number check as a 5th usage site made this measurably worse, especially in the worst case where a segment has zero pre-verified numbers and the scan can't terminate early via LIMIT (confirmed via reconstructing the pre-Task-3 query on an identical fixture: that scenario already cost ~3.4s before this feature, and the 5th usage site pushed it to 4.1-9.5s with real variance, including an outright failure over 5 consecutive runs). Restructured candidateQuery to compute lead_number/lead_equivalent once per row in a derived subquery (aliased `candidates`), referenced by column everywhere else instead of re-embedding the expression text. SQLite still flattens/uses idx_leads_campaign_order for the segmento scan and sort (verified via EXPLAIN QUERY PLAN), so the common fast path is unaffected (~2ms), while the worst-case path dropped to a consistent ~3-5s across 5 consecutive runs of the 1.3M-row perf test. --- src/outreach/campaign-queue.mjs | 50 +++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/src/outreach/campaign-queue.mjs b/src/outreach/campaign-queue.mjs index a91ea94..0597aa9 100644 --- a/src/outreach/campaign-queue.mjs +++ b/src/outreach/campaign-queue.mjs @@ -25,18 +25,37 @@ function equivalentSql(column) { } function candidateQuery(confirmed) { - const leadNumber = normalizedSql('leads.whatsapp'); - const leadEquivalent = equivalentSql('leads.whatsapp'); + // leadNumber/leadEquivalent are computed ONCE per row in the derived `candidates` + // subquery below (as lead_number/lead_equivalent columns) instead of being + // re-embedded as raw expression text at every usage site further down. The + // normalizedSql()/equivalentSql() expressions expand into a deeply nested chain + // of replace()/substr()/length() calls; textually re-embedding them at each of + // this query's several correlated-subquery sites (verification join, opt-out + // check, shared-number check, inbound-replies check) made SQLite recompute that + // whole chain independently at every site, for every row — measurably expensive + // once a 5th usage site (the shared-number check) was added, especially in the + // worst case where a candidate segment has zero pre-verified numbers and the + // scan can't terminate early (see task-3-report.md for the investigation). + const leadNumberExpr = normalizedSql('raw.whatsapp'); + const leadEquivalentExpr = equivalentSql('raw.whatsapp'); + const leadNumber = 'candidates.lead_number'; + const leadEquivalent = 'candidates.lead_equivalent'; const verificationFilter = confirmed ? "verification.status = 'confirmado'" : "(verification.status IS NULL OR verification.status = 'erro_temporario')"; return `/* campaign-candidates */ - SELECT leads.id, leads.empresa, leads.legal_name, leads.whatsapp, leads.segmento, - leads.prioridade, leads.score, + SELECT candidates.id, candidates.empresa, candidates.legal_name, candidates.whatsapp, candidates.segmento, + candidates.prioridade, candidates.score, verification.status AS verification_status, verification.numero AS verification_number, verification.numero_resolvido AS verification_resolved_number - FROM leads + FROM ( + SELECT raw.*, + ${leadNumberExpr} AS lead_number, + ${leadEquivalentExpr} AS lead_equivalent + FROM leads raw + WHERE raw.segmento = ? + ) AS candidates LEFT JOIN whatsapp_number_verifications verification ON verification.rowid = ( SELECT latest.rowid @@ -45,17 +64,16 @@ function candidateQuery(confirmed) { ORDER BY latest.checked_at DESC, latest.rowid DESC LIMIT 1 ) - WHERE leads.segmento = ? - AND ${leadNumber} <> '' + WHERE ${leadNumber} <> '' AND ${verificationFilter} AND NOT EXISTS ( SELECT 1 FROM outreach_sends sends - WHERE sends.lead_id = leads.id AND sends.produto = ? + WHERE sends.lead_id = candidates.id AND sends.produto = ? ) AND NOT EXISTS ( SELECT 1 FROM outreach_campaign_queue queued - WHERE queued.lead_id = leads.id + WHERE queued.lead_id = candidates.id AND queued.produto = ? AND queued.status IN (${ACTIVE_QUEUE_STATUSES}) ) @@ -64,9 +82,11 @@ function candidateQuery(confirmed) { WHERE opt_out.numero IN (${leadNumber}, ${leadEquivalent}) ) AND ( - (SELECT COUNT(DISTINCT other.id) FROM leads other - WHERE other.whatsapp IN (${leadNumber}, ${leadEquivalent}) - ) < 2 + NOT EXISTS ( + SELECT 1 FROM leads other + WHERE other.whatsapp IN (${leadNumber}, ${leadEquivalent}) + AND other.id <> candidates.id + ) OR EXISTS ( SELECT 1 FROM outreach_shared_number_overrides ov WHERE ov.numero IN (${leadNumber}, ${leadEquivalent}) @@ -74,14 +94,14 @@ function candidateQuery(confirmed) { ) AND NOT EXISTS ( SELECT 1 FROM outreach_inbound_replies replies - WHERE replies.lead_id = leads.id + WHERE replies.lead_id = candidates.id OR replies.numero IN (${leadNumber}, ${leadEquivalent}) ) AND NOT EXISTS ( SELECT 1 FROM outreach_lead_states states - WHERE states.lead_id = leads.id AND states.first_reply_at IS NOT NULL + WHERE states.lead_id = candidates.id AND states.first_reply_at IS NOT NULL ) - ORDER BY leads.prioridade ASC, leads.score DESC, leads.id ASC + ORDER BY candidates.prioridade ASC, candidates.score DESC, candidates.id ASC LIMIT ?`; }