Fix perf flakiness in shared-number candidate query by precomputing lead numbers once

The candidate query's lead_number/lead_equivalent forms were being
re-embedded as raw SQL expression text at every usage site (verification
join, opt-out check, shared-number check, inbound-replies check). Each
embedding re-evaluates the same deeply nested replace()/substr()/length()
chain per row; adding the shared-number check as a 5th usage site made
this measurably worse, especially in the worst case where a segment has
zero pre-verified numbers and the scan can't terminate early via LIMIT
(confirmed via reconstructing the pre-Task-3 query on an identical
fixture: that scenario already cost ~3.4s before this feature, and the
5th usage site pushed it to 4.1-9.5s with real variance, including an
outright failure over 5 consecutive runs).

Restructured candidateQuery to compute lead_number/lead_equivalent once
per row in a derived subquery (aliased `candidates`), referenced by
column everywhere else instead of re-embedding the expression text.
SQLite still flattens/uses idx_leads_campaign_order for the segmento
scan and sort (verified via EXPLAIN QUERY PLAN), so the common fast path
is unaffected (~2ms), while the worst-case path dropped to a consistent
~3-5s across 5 consecutive runs of the 1.3M-row perf test.
This commit is contained in:
Gustavo
2026-07-31 11:52:48 -03:00
parent d85fb12991
commit e9f585458f
+35 -15
View File
@@ -25,18 +25,37 @@ function equivalentSql(column) {
}
function candidateQuery(confirmed) {
const leadNumber = normalizedSql('leads.whatsapp');
const leadEquivalent = equivalentSql('leads.whatsapp');
// leadNumber/leadEquivalent are computed ONCE per row in the derived `candidates`
// subquery below (as lead_number/lead_equivalent columns) instead of being
// re-embedded as raw expression text at every usage site further down. The
// normalizedSql()/equivalentSql() expressions expand into a deeply nested chain
// of replace()/substr()/length() calls; textually re-embedding them at each of
// this query's several correlated-subquery sites (verification join, opt-out
// check, shared-number check, inbound-replies check) made SQLite recompute that
// whole chain independently at every site, for every row — measurably expensive
// once a 5th usage site (the shared-number check) was added, especially in the
// worst case where a candidate segment has zero pre-verified numbers and the
// scan can't terminate early (see task-3-report.md for the investigation).
const leadNumberExpr = normalizedSql('raw.whatsapp');
const leadEquivalentExpr = equivalentSql('raw.whatsapp');
const leadNumber = 'candidates.lead_number';
const leadEquivalent = 'candidates.lead_equivalent';
const verificationFilter = confirmed
? "verification.status = 'confirmado'"
: "(verification.status IS NULL OR verification.status = 'erro_temporario')";
return `/* campaign-candidates */
SELECT leads.id, leads.empresa, leads.legal_name, leads.whatsapp, leads.segmento,
leads.prioridade, leads.score,
SELECT candidates.id, candidates.empresa, candidates.legal_name, candidates.whatsapp, candidates.segmento,
candidates.prioridade, candidates.score,
verification.status AS verification_status,
verification.numero AS verification_number,
verification.numero_resolvido AS verification_resolved_number
FROM leads
FROM (
SELECT raw.*,
${leadNumberExpr} AS lead_number,
${leadEquivalentExpr} AS lead_equivalent
FROM leads raw
WHERE raw.segmento = ?
) AS candidates
LEFT JOIN whatsapp_number_verifications verification
ON verification.rowid = (
SELECT latest.rowid
@@ -45,17 +64,16 @@ function candidateQuery(confirmed) {
ORDER BY latest.checked_at DESC, latest.rowid DESC
LIMIT 1
)
WHERE leads.segmento = ?
AND ${leadNumber} <> ''
WHERE ${leadNumber} <> ''
AND ${verificationFilter}
AND NOT EXISTS (
SELECT 1 FROM outreach_sends sends
WHERE sends.lead_id = leads.id AND sends.produto = ?
WHERE sends.lead_id = candidates.id AND sends.produto = ?
)
AND NOT EXISTS (
SELECT 1
FROM outreach_campaign_queue queued
WHERE queued.lead_id = leads.id
WHERE queued.lead_id = candidates.id
AND queued.produto = ?
AND queued.status IN (${ACTIVE_QUEUE_STATUSES})
)
@@ -64,9 +82,11 @@ function candidateQuery(confirmed) {
WHERE opt_out.numero IN (${leadNumber}, ${leadEquivalent})
)
AND (
(SELECT COUNT(DISTINCT other.id) FROM leads other
WHERE other.whatsapp IN (${leadNumber}, ${leadEquivalent})
) < 2
NOT EXISTS (
SELECT 1 FROM leads other
WHERE other.whatsapp IN (${leadNumber}, ${leadEquivalent})
AND other.id <> candidates.id
)
OR EXISTS (
SELECT 1 FROM outreach_shared_number_overrides ov
WHERE ov.numero IN (${leadNumber}, ${leadEquivalent})
@@ -74,14 +94,14 @@ function candidateQuery(confirmed) {
)
AND NOT EXISTS (
SELECT 1 FROM outreach_inbound_replies replies
WHERE replies.lead_id = leads.id
WHERE replies.lead_id = candidates.id
OR replies.numero IN (${leadNumber}, ${leadEquivalent})
)
AND NOT EXISTS (
SELECT 1 FROM outreach_lead_states states
WHERE states.lead_id = leads.id AND states.first_reply_at IS NOT NULL
WHERE states.lead_id = candidates.id AND states.first_reply_at IS NOT NULL
)
ORDER BY leads.prioridade ASC, leads.score DESC, leads.id ASC
ORDER BY candidates.prioridade ASC, candidates.score DESC, candidates.id ASC
LIMIT ?`;
}