Fix perf flakiness in shared-number candidate query by precomputing lead numbers once

The candidate query's lead_number/lead_equivalent forms were being
re-embedded as raw SQL expression text at every usage site (verification
join, opt-out check, shared-number check, inbound-replies check). Each
embedding re-evaluates the same deeply nested replace()/substr()/length()
chain per row; adding the shared-number check as a 5th usage site made
this measurably worse, especially in the worst case where a segment has
zero pre-verified numbers and the scan can't terminate early via LIMIT
(confirmed via reconstructing the pre-Task-3 query on an identical
fixture: that scenario already cost ~3.4s before this feature, and the
5th usage site pushed it to 4.1-9.5s with real variance, including an
outright failure over 5 consecutive runs).

Restructured candidateQuery to compute lead_number/lead_equivalent once
per row in a derived subquery (aliased `candidates`), referenced by
column everywhere else instead of re-embedding the expression text.
SQLite still flattens/uses idx_leads_campaign_order for the segmento
scan and sort (verified via EXPLAIN QUERY PLAN), so the common fast path
is unaffected (~2ms), while the worst-case path dropped to a consistent
~3-5s across 5 consecutive runs of the 1.3M-row perf test.
This commit is contained in:
Gustavo
2026-07-31 11:52:48 -03:00
parent d85fb12991
commit e9f585458f
+34 -14
View File
@@ -25,18 +25,37 @@ function equivalentSql(column) {
} }
function candidateQuery(confirmed) { function candidateQuery(confirmed) {
const leadNumber = normalizedSql('leads.whatsapp'); // leadNumber/leadEquivalent are computed ONCE per row in the derived `candidates`
const leadEquivalent = equivalentSql('leads.whatsapp'); // subquery below (as lead_number/lead_equivalent columns) instead of being
// re-embedded as raw expression text at every usage site further down. The
// normalizedSql()/equivalentSql() expressions expand into a deeply nested chain
// of replace()/substr()/length() calls; textually re-embedding them at each of
// this query's several correlated-subquery sites (verification join, opt-out
// check, shared-number check, inbound-replies check) made SQLite recompute that
// whole chain independently at every site, for every row — measurably expensive
// once a 5th usage site (the shared-number check) was added, especially in the
// worst case where a candidate segment has zero pre-verified numbers and the
// scan can't terminate early (see task-3-report.md for the investigation).
const leadNumberExpr = normalizedSql('raw.whatsapp');
const leadEquivalentExpr = equivalentSql('raw.whatsapp');
const leadNumber = 'candidates.lead_number';
const leadEquivalent = 'candidates.lead_equivalent';
const verificationFilter = confirmed const verificationFilter = confirmed
? "verification.status = 'confirmado'" ? "verification.status = 'confirmado'"
: "(verification.status IS NULL OR verification.status = 'erro_temporario')"; : "(verification.status IS NULL OR verification.status = 'erro_temporario')";
return `/* campaign-candidates */ return `/* campaign-candidates */
SELECT leads.id, leads.empresa, leads.legal_name, leads.whatsapp, leads.segmento, SELECT candidates.id, candidates.empresa, candidates.legal_name, candidates.whatsapp, candidates.segmento,
leads.prioridade, leads.score, candidates.prioridade, candidates.score,
verification.status AS verification_status, verification.status AS verification_status,
verification.numero AS verification_number, verification.numero AS verification_number,
verification.numero_resolvido AS verification_resolved_number verification.numero_resolvido AS verification_resolved_number
FROM leads FROM (
SELECT raw.*,
${leadNumberExpr} AS lead_number,
${leadEquivalentExpr} AS lead_equivalent
FROM leads raw
WHERE raw.segmento = ?
) AS candidates
LEFT JOIN whatsapp_number_verifications verification LEFT JOIN whatsapp_number_verifications verification
ON verification.rowid = ( ON verification.rowid = (
SELECT latest.rowid SELECT latest.rowid
@@ -45,17 +64,16 @@ function candidateQuery(confirmed) {
ORDER BY latest.checked_at DESC, latest.rowid DESC ORDER BY latest.checked_at DESC, latest.rowid DESC
LIMIT 1 LIMIT 1
) )
WHERE leads.segmento = ? WHERE ${leadNumber} <> ''
AND ${leadNumber} <> ''
AND ${verificationFilter} AND ${verificationFilter}
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM outreach_sends sends SELECT 1 FROM outreach_sends sends
WHERE sends.lead_id = leads.id AND sends.produto = ? WHERE sends.lead_id = candidates.id AND sends.produto = ?
) )
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 SELECT 1
FROM outreach_campaign_queue queued FROM outreach_campaign_queue queued
WHERE queued.lead_id = leads.id WHERE queued.lead_id = candidates.id
AND queued.produto = ? AND queued.produto = ?
AND queued.status IN (${ACTIVE_QUEUE_STATUSES}) AND queued.status IN (${ACTIVE_QUEUE_STATUSES})
) )
@@ -64,9 +82,11 @@ function candidateQuery(confirmed) {
WHERE opt_out.numero IN (${leadNumber}, ${leadEquivalent}) WHERE opt_out.numero IN (${leadNumber}, ${leadEquivalent})
) )
AND ( AND (
(SELECT COUNT(DISTINCT other.id) FROM leads other NOT EXISTS (
SELECT 1 FROM leads other
WHERE other.whatsapp IN (${leadNumber}, ${leadEquivalent}) WHERE other.whatsapp IN (${leadNumber}, ${leadEquivalent})
) < 2 AND other.id <> candidates.id
)
OR EXISTS ( OR EXISTS (
SELECT 1 FROM outreach_shared_number_overrides ov SELECT 1 FROM outreach_shared_number_overrides ov
WHERE ov.numero IN (${leadNumber}, ${leadEquivalent}) WHERE ov.numero IN (${leadNumber}, ${leadEquivalent})
@@ -74,14 +94,14 @@ function candidateQuery(confirmed) {
) )
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM outreach_inbound_replies replies SELECT 1 FROM outreach_inbound_replies replies
WHERE replies.lead_id = leads.id WHERE replies.lead_id = candidates.id
OR replies.numero IN (${leadNumber}, ${leadEquivalent}) OR replies.numero IN (${leadNumber}, ${leadEquivalent})
) )
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM outreach_lead_states states SELECT 1 FROM outreach_lead_states states
WHERE states.lead_id = leads.id AND states.first_reply_at IS NOT NULL WHERE states.lead_id = candidates.id AND states.first_reply_at IS NOT NULL
) )
ORDER BY leads.prioridade ASC, leads.score DESC, leads.id ASC ORDER BY candidates.prioridade ASC, candidates.score DESC, candidates.id ASC
LIMIT ?`; LIMIT ?`;
} }