Fix perf flakiness in shared-number candidate query by precomputing lead numbers once
The candidate query's lead_number/lead_equivalent forms were being re-embedded as raw SQL expression text at every usage site (verification join, opt-out check, shared-number check, inbound-replies check). Each embedding re-evaluates the same deeply nested replace()/substr()/length() chain per row; adding the shared-number check as a 5th usage site made this measurably worse, especially in the worst case where a segment has zero pre-verified numbers and the scan can't terminate early via LIMIT (confirmed via reconstructing the pre-Task-3 query on an identical fixture: that scenario already cost ~3.4s before this feature, and the 5th usage site pushed it to 4.1-9.5s with real variance, including an outright failure over 5 consecutive runs). Restructured candidateQuery to compute lead_number/lead_equivalent once per row in a derived subquery (aliased `candidates`), referenced by column everywhere else instead of re-embedding the expression text. SQLite still flattens/uses idx_leads_campaign_order for the segmento scan and sort (verified via EXPLAIN QUERY PLAN), so the common fast path is unaffected (~2ms), while the worst-case path dropped to a consistent ~3-5s across 5 consecutive runs of the 1.3M-row perf test.
This commit is contained in:
@@ -25,18 +25,37 @@ function equivalentSql(column) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function candidateQuery(confirmed) {
|
function candidateQuery(confirmed) {
|
||||||
const leadNumber = normalizedSql('leads.whatsapp');
|
// leadNumber/leadEquivalent are computed ONCE per row in the derived `candidates`
|
||||||
const leadEquivalent = equivalentSql('leads.whatsapp');
|
// subquery below (as lead_number/lead_equivalent columns) instead of being
|
||||||
|
// re-embedded as raw expression text at every usage site further down. The
|
||||||
|
// normalizedSql()/equivalentSql() expressions expand into a deeply nested chain
|
||||||
|
// of replace()/substr()/length() calls; textually re-embedding them at each of
|
||||||
|
// this query's several correlated-subquery sites (verification join, opt-out
|
||||||
|
// check, shared-number check, inbound-replies check) made SQLite recompute that
|
||||||
|
// whole chain independently at every site, for every row — measurably expensive
|
||||||
|
// once a 5th usage site (the shared-number check) was added, especially in the
|
||||||
|
// worst case where a candidate segment has zero pre-verified numbers and the
|
||||||
|
// scan can't terminate early (see task-3-report.md for the investigation).
|
||||||
|
const leadNumberExpr = normalizedSql('raw.whatsapp');
|
||||||
|
const leadEquivalentExpr = equivalentSql('raw.whatsapp');
|
||||||
|
const leadNumber = 'candidates.lead_number';
|
||||||
|
const leadEquivalent = 'candidates.lead_equivalent';
|
||||||
const verificationFilter = confirmed
|
const verificationFilter = confirmed
|
||||||
? "verification.status = 'confirmado'"
|
? "verification.status = 'confirmado'"
|
||||||
: "(verification.status IS NULL OR verification.status = 'erro_temporario')";
|
: "(verification.status IS NULL OR verification.status = 'erro_temporario')";
|
||||||
return `/* campaign-candidates */
|
return `/* campaign-candidates */
|
||||||
SELECT leads.id, leads.empresa, leads.legal_name, leads.whatsapp, leads.segmento,
|
SELECT candidates.id, candidates.empresa, candidates.legal_name, candidates.whatsapp, candidates.segmento,
|
||||||
leads.prioridade, leads.score,
|
candidates.prioridade, candidates.score,
|
||||||
verification.status AS verification_status,
|
verification.status AS verification_status,
|
||||||
verification.numero AS verification_number,
|
verification.numero AS verification_number,
|
||||||
verification.numero_resolvido AS verification_resolved_number
|
verification.numero_resolvido AS verification_resolved_number
|
||||||
FROM leads
|
FROM (
|
||||||
|
SELECT raw.*,
|
||||||
|
${leadNumberExpr} AS lead_number,
|
||||||
|
${leadEquivalentExpr} AS lead_equivalent
|
||||||
|
FROM leads raw
|
||||||
|
WHERE raw.segmento = ?
|
||||||
|
) AS candidates
|
||||||
LEFT JOIN whatsapp_number_verifications verification
|
LEFT JOIN whatsapp_number_verifications verification
|
||||||
ON verification.rowid = (
|
ON verification.rowid = (
|
||||||
SELECT latest.rowid
|
SELECT latest.rowid
|
||||||
@@ -45,17 +64,16 @@ function candidateQuery(confirmed) {
|
|||||||
ORDER BY latest.checked_at DESC, latest.rowid DESC
|
ORDER BY latest.checked_at DESC, latest.rowid DESC
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
)
|
)
|
||||||
WHERE leads.segmento = ?
|
WHERE ${leadNumber} <> ''
|
||||||
AND ${leadNumber} <> ''
|
|
||||||
AND ${verificationFilter}
|
AND ${verificationFilter}
|
||||||
AND NOT EXISTS (
|
AND NOT EXISTS (
|
||||||
SELECT 1 FROM outreach_sends sends
|
SELECT 1 FROM outreach_sends sends
|
||||||
WHERE sends.lead_id = leads.id AND sends.produto = ?
|
WHERE sends.lead_id = candidates.id AND sends.produto = ?
|
||||||
)
|
)
|
||||||
AND NOT EXISTS (
|
AND NOT EXISTS (
|
||||||
SELECT 1
|
SELECT 1
|
||||||
FROM outreach_campaign_queue queued
|
FROM outreach_campaign_queue queued
|
||||||
WHERE queued.lead_id = leads.id
|
WHERE queued.lead_id = candidates.id
|
||||||
AND queued.produto = ?
|
AND queued.produto = ?
|
||||||
AND queued.status IN (${ACTIVE_QUEUE_STATUSES})
|
AND queued.status IN (${ACTIVE_QUEUE_STATUSES})
|
||||||
)
|
)
|
||||||
@@ -64,9 +82,11 @@ function candidateQuery(confirmed) {
|
|||||||
WHERE opt_out.numero IN (${leadNumber}, ${leadEquivalent})
|
WHERE opt_out.numero IN (${leadNumber}, ${leadEquivalent})
|
||||||
)
|
)
|
||||||
AND (
|
AND (
|
||||||
(SELECT COUNT(DISTINCT other.id) FROM leads other
|
NOT EXISTS (
|
||||||
WHERE other.whatsapp IN (${leadNumber}, ${leadEquivalent})
|
SELECT 1 FROM leads other
|
||||||
) < 2
|
WHERE other.whatsapp IN (${leadNumber}, ${leadEquivalent})
|
||||||
|
AND other.id <> candidates.id
|
||||||
|
)
|
||||||
OR EXISTS (
|
OR EXISTS (
|
||||||
SELECT 1 FROM outreach_shared_number_overrides ov
|
SELECT 1 FROM outreach_shared_number_overrides ov
|
||||||
WHERE ov.numero IN (${leadNumber}, ${leadEquivalent})
|
WHERE ov.numero IN (${leadNumber}, ${leadEquivalent})
|
||||||
@@ -74,14 +94,14 @@ function candidateQuery(confirmed) {
|
|||||||
)
|
)
|
||||||
AND NOT EXISTS (
|
AND NOT EXISTS (
|
||||||
SELECT 1 FROM outreach_inbound_replies replies
|
SELECT 1 FROM outreach_inbound_replies replies
|
||||||
WHERE replies.lead_id = leads.id
|
WHERE replies.lead_id = candidates.id
|
||||||
OR replies.numero IN (${leadNumber}, ${leadEquivalent})
|
OR replies.numero IN (${leadNumber}, ${leadEquivalent})
|
||||||
)
|
)
|
||||||
AND NOT EXISTS (
|
AND NOT EXISTS (
|
||||||
SELECT 1 FROM outreach_lead_states states
|
SELECT 1 FROM outreach_lead_states states
|
||||||
WHERE states.lead_id = leads.id AND states.first_reply_at IS NOT NULL
|
WHERE states.lead_id = candidates.id AND states.first_reply_at IS NOT NULL
|
||||||
)
|
)
|
||||||
ORDER BY leads.prioridade ASC, leads.score DESC, leads.id ASC
|
ORDER BY candidates.prioridade ASC, candidates.score DESC, candidates.id ASC
|
||||||
LIMIT ?`;
|
LIMIT ?`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user