diff --git a/.env.self-host.example b/.env.self-host.example index 557adbd15..3df0b6219 100644 --- a/.env.self-host.example +++ b/.env.self-host.example @@ -42,6 +42,9 @@ USE_HTTPS=false AWS_SES_REGION=us-east-1 AWS_SES_ACCESS_KEY_ID= AWS_SES_SECRET_ACCESS_KEY= +# Exact SNS topics allowed to call /webhooks/sns. Include a separate inbound +# topic too, if used, as a comma-separated ARN. +SNS_TOPIC_ARNS=arn:aws:sns:us-east-1:123456789012:plunk-ses-events # Configuration sets for email tracking # SES_CONFIGURATION_SET: Default configuration with open/click tracking enabled diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85f9ee795..3f2c710f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,6 +110,7 @@ jobs: AWS_SES_REGION=us-east-1 AWS_SES_ACCESS_KEY_ID=mock AWS_SES_SECRET_ACCESS_KEY=mock + SNS_TOPIC_ARNS=arn:aws:sns:us-east-1:123456789012:plunk-ses-events SES_CONFIGURATION_SET=test SES_CONFIGURATION_SET_NO_TRACKING=test-no-tracking EOF @@ -195,6 +196,7 @@ jobs: AWS_SES_REGION=us-east-1 AWS_SES_ACCESS_KEY_ID=mock AWS_SES_SECRET_ACCESS_KEY=mock + SNS_TOPIC_ARNS=arn:aws:sns:us-east-1:123456789012:plunk-ses-events SES_CONFIGURATION_SET=test SES_CONFIGURATION_SET_NO_TRACKING=test EOF diff --git a/CLAUDE.md b/CLAUDE.md index deb33eb28..01489b879 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -140,8 +140,8 @@ Required for builds and deployment (see turbo.json and .env.example): optional) - S3-compatible Storage (Minio): `S3_ENDPOINT`, `S3_ACCESS_KEY_ID`, `S3_ACCESS_KEY_SECRET`, `S3_BUCKET`, `S3_PUBLIC_URL`, `S3_FORCE_PATH_STYLE` -- AWS SES: `AWS_SES_REGION`, `AWS_SES_ACCESS_KEY_ID`, `AWS_SES_SECRET_ACCESS_KEY`, `SES_CONFIGURATION_SET`, - `SES_CONFIGURATION_SET_NO_TRACKING` +- AWS SES: `AWS_SES_REGION`, `AWS_SES_ACCESS_KEY_ID`, `AWS_SES_SECRET_ACCESS_KEY`, `SNS_TOPIC_ARNS`, + `SES_CONFIGURATION_SET`, `SES_CONFIGURATION_SET_NO_TRACKING` - OAuth (optional): `GITHUB_OAUTH_CLIENT`, `GITHUB_OAUTH_SECRET`, `GOOGLE_OAUTH_CLIENT`, `GOOGLE_OAUTH_SECRET` - Stripe (optional): `STRIPE_SK`, `STRIPE_WEBHOOK_SECRET`, `STRIPE_PRICE_ONBOARDING`, `STRIPE_PRICE_EMAIL_USAGE`, `STRIPE_METER_EVENT_NAME` diff --git a/apps/api/.env.example b/apps/api/.env.example index 96966897f..ea94ad5eb 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -54,6 +54,8 @@ S3_FORCE_PATH_STYLE=true AWS_SES_REGION=eu-north-1 AWS_SES_ACCESS_KEY_ID= AWS_SES_SECRET_ACCESS_KEY= +# Exact SNS topics allowed to call /webhooks/sns (comma-separated when using more than one) +SNS_TOPIC_ARNS=arn:aws:sns:eu-north-1:123456789012:plunk-ses-events # Configuration sets for email tracking SES_CONFIGURATION_SET=plunk-configuration-set # Default: with open/click tracking diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index f86c10781..d5f188443 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -523,20 +523,23 @@ void prisma.$connect().then(async () => { signale.info('[BACKGROUND-JOB] API request cleanup scheduled (BullMQ repeatable job, runs daily at 3 AM)'); - // Set up repeatable job for expired idempotency key cleanup (BullMQ) - // Runs hourly: expiry is what makes a key reusable, so the sweep should track the TTL - await idempotencyKeyCleanupQueue.add( + // Set up repeatable durable-state maintenance (BullMQ). Event dispatch uses a + // five-minute grace window, so a minutely sweep retries failed ingestion in + // under six minutes during normal operation without racing live requests. + // v0.14.0 registered this job hourly through BullMQ's legacy repeat API. + // Remove that exact entry before moving the stable scheduler to one minute. + await idempotencyKeyCleanupQueue.removeRepeatable( 'cleanup-expired-keys', - {}, - { - repeat: { - pattern: '0 * * * *', // Hourly, on the hour - }, - jobId: 'idempotency-key-cleanup-repeatable', // Fixed ID to prevent duplicates - }, + {pattern: '0 * * * *'}, + 'idempotency-key-cleanup-repeatable', + ); + await idempotencyKeyCleanupQueue.upsertJobScheduler( + 'idempotency-key-cleanup-repeatable', + {pattern: '* * * * *'}, + {name: 'cleanup-expired-keys', data: {}}, ); - signale.info('[BACKGROUND-JOB] Idempotency key cleanup scheduled (BullMQ repeatable job, runs hourly)'); + signale.info('[BACKGROUND-JOB] Durable-state maintenance scheduled (BullMQ repeatable job, runs every minute)'); // Set up repeatable job for email body cleanup (BullMQ) // Run daily at 4 AM, offset from the API request cleanup so the two don't overlap diff --git a/apps/api/src/app/constants.ts b/apps/api/src/app/constants.ts index b2299776b..70e393bca 100644 --- a/apps/api/src/app/constants.ts +++ b/apps/api/src/app/constants.ts @@ -45,6 +45,19 @@ export const AWS_SES_REGION = validateEnv('AWS_SES_REGION'); export const AWS_SES_ACCESS_KEY_ID = validateEnv('AWS_SES_ACCESS_KEY_ID'); export const AWS_SES_SECRET_ACCESS_KEY = validateEnv('AWS_SES_SECRET_ACCESS_KEY'); +// Exact SNS topic ARNs authorized to deliver SES events to /webhooks/sns. +// Multiple topics support deployments that separate outbound and inbound SES. +const snsTopicArns = validateEnv('SNS_TOPIC_ARNS') + .split(',') + .map(topicArn => topicArn.trim()) + .filter(Boolean); + +if (snsTopicArns.length === 0) { + throw new Error('SNS_TOPIC_ARNS must contain at least one topic ARN'); +} + +export const SNS_TOPIC_ARNS: ReadonlySet = new Set(snsTopicArns); + // Custom MAIL FROM subdomain used to construct `.` // when a domain is added. Defaults to `plunk`. Override when `plunk.` // is already used for something else (e.g. a CDN), since the MAIL FROM hostname diff --git a/apps/api/src/controllers/Actions.ts b/apps/api/src/controllers/Actions.ts index e3b7d3a36..39e9c964c 100644 --- a/apps/api/src/controllers/Actions.ts +++ b/apps/api/src/controllers/Actions.ts @@ -39,6 +39,8 @@ export class Actions { * Response: * - success: boolean * - data: object with contact ID, event ID, and timestamp + * - 200 once the event is stored; workflow dispatch may finish synchronously + * or through Plunk's bounded internal reconciliation sweep * * Example: * { diff --git a/apps/api/src/controllers/Webhooks.ts b/apps/api/src/controllers/Webhooks.ts index 65045717c..9731c7656 100644 --- a/apps/api/src/controllers/Webhooks.ts +++ b/apps/api/src/controllers/Webhooks.ts @@ -1,6 +1,9 @@ +import {randomUUID} from 'node:crypto'; + import {Controller, Post} from '@overnightjs/core'; import type {Prisma} from '@plunk/db'; import {EmailSourceType, EmailStatus} from '@plunk/db'; +import {toPrismaJson} from '@plunk/types'; import type {Request, Response} from 'express'; import {simpleParser} from 'mailparser'; import sanitizeHtml from 'sanitize-html'; @@ -24,6 +27,157 @@ import {QueueService} from '../services/QueueService.js'; import {SecurityService} from '../services/SecurityService.js'; import {CatchAsync} from '../utils/asyncHandler.js'; +const SNS_CLAIM_LEASE_MS = 5 * 60 * 1000; +const SNS_CLAIM_HEARTBEAT_MS = 60 * 1000; +const SNS_CLAIM_ATTEMPTS = 3; +const SNS_RECEIPT_TTL_MS = 7 * 24 * 60 * 60 * 1000; + +type ActiveSnsClaim = { + messageId: string; + processingToken: string; +}; + +type SnsReceiptClient = Pick; + +type SnsClaimResult = + | {outcome: 'claimed'; claim: ActiveSnsClaim} + | {outcome: 'completed'} + | {outcome: 'in-flight'}; + +function isUniqueConstraintError(error: unknown): boolean { + return error instanceof Error && 'code' in error && error.code === 'P2002'; +} + +/** + * Claim a signed SNS MessageId before applying its side effects. + * + * The unique index decides concurrent races. Completed deliveries are safe to + * acknowledge, active deliveries receive a retryable response, and failed or + * abandoned deliveries are reclaimed with a compare-and-swap update. + */ +async function claimSnsNotification(messageId: string): Promise { + for (let attempt = 0; attempt < SNS_CLAIM_ATTEMPTS; attempt++) { + const processingToken = randomUUID(); + const processingStartedAt = new Date(); + + try { + await prisma.snsWebhookReceipt.create({ + data: { + messageId, + processingToken, + processingStartedAt, + expiresAt: new Date(processingStartedAt.getTime() + SNS_RECEIPT_TTL_MS), + }, + }); + return {outcome: 'claimed', claim: {messageId, processingToken}}; + } catch (error) { + if (!isUniqueConstraintError(error)) { + throw error; + } + } + + const existing = await prisma.snsWebhookReceipt.findUnique({ + where: {messageId}, + select: {status: true, processingToken: true, processingStartedAt: true}, + }); + + // The prior row may have been released between the unique conflict and read. + if (!existing) { + continue; + } + + if (existing.status === 'COMPLETED') { + return {outcome: 'completed'}; + } + + const leaseCutoff = Date.now() - SNS_CLAIM_LEASE_MS; + if (existing.status === 'PROCESSING' && existing.processingStartedAt.getTime() > leaseCutoff) { + return {outcome: 'in-flight'}; + } + + const reclaimed = await prisma.snsWebhookReceipt.updateMany({ + where: { + messageId, + status: existing.status, + processingToken: existing.processingToken, + processingStartedAt: existing.processingStartedAt, + }, + data: { + status: 'PROCESSING', + processingToken, + processingStartedAt, + completedAt: null, + }, + }); + + if (reclaimed.count === 1) { + return {outcome: 'claimed', claim: {messageId, processingToken}}; + } + } + + // Repeated claim races are transient. Do not acknowledge until one delivery + // has durably recorded completion. + return {outcome: 'in-flight'}; +} + +async function completeSnsNotification( + claim: ActiveSnsClaim, + client: SnsReceiptClient = prisma, +): Promise { + const completed = await client.snsWebhookReceipt.updateMany({ + where: { + messageId: claim.messageId, + processingToken: claim.processingToken, + status: 'PROCESSING', + }, + data: {status: 'COMPLETED', completedAt: new Date()}, + }); + + if (completed.count !== 1) { + throw new Error(`SNS claim ${claim.messageId} could not be completed`); + } +} + +async function failSnsNotification(claim: ActiveSnsClaim): Promise { + const failed = await prisma.snsWebhookReceipt.updateMany({ + where: { + messageId: claim.messageId, + processingToken: claim.processingToken, + status: 'PROCESSING', + }, + data: {status: 'FAILED'}, + }); + + if (failed.count !== 1) { + signale.warn(`[WEBHOOK] SNS claim ${claim.messageId} was no longer active while recording failure`); + } +} + +function startSnsClaimHeartbeat(claim: ActiveSnsClaim): () => void { + const timer = setInterval(() => { + void prisma.snsWebhookReceipt + .updateMany({ + where: { + messageId: claim.messageId, + processingToken: claim.processingToken, + status: 'PROCESSING', + }, + data: {processingStartedAt: new Date()}, + }) + .then(renewed => { + if (renewed.count !== 1) { + signale.warn(`[WEBHOOK] SNS claim ${claim.messageId} was no longer active during heartbeat`); + } + }) + .catch(error => { + signale.error(`[WEBHOOK] Failed to renew SNS claim ${claim.messageId}:`, error); + }); + }, SNS_CLAIM_HEARTBEAT_MS); + + timer.unref(); + return () => clearInterval(timer); +} + /** * Webhooks Controller * Handles incoming webhooks from external services (AWS SNS/SES) @@ -38,6 +192,16 @@ export class Webhooks { @Post('sns') @CatchAsync public async receiveSNSWebhook(req: Request, res: Response) { + let activeSnsClaim: ActiveSnsClaim | undefined; + let stopSnsClaimHeartbeat: (() => void) | undefined; + + const completeActiveClaim = async () => { + if (!activeSnsClaim) return; + + await completeSnsNotification(activeSnsClaim); + activeSnsClaim = undefined; + }; + try { // Verify SNS message signature before processing anything const signatureValid = await SecurityService.verifySnsSignature(req.body as Record); @@ -89,14 +253,14 @@ export class Webhooks { }); } else { signale.error('Failed to confirm SNS subscription:', confirmResponse.statusText); - return res.status(200).json({ + return res.status(502).json({ success: false, message: 'Failed to confirm subscription', }); } } catch (confirmError) { signale.error('Error confirming SNS subscription:', confirmError); - return res.status(200).json({ + return res.status(502).json({ success: false, message: 'Error confirming subscription', }); @@ -109,23 +273,75 @@ export class Webhooks { return res.status(200).json({success: false, message: 'Unknown message type'}); } + const snsMessageId: unknown = req.body.MessageId; + if (typeof snsMessageId !== 'string' || snsMessageId.length === 0) { + signale.warn('[WEBHOOK] SNS notification missing MessageId'); + return res.status(400).json({success: false, message: 'Missing SNS MessageId'}); + } + // Parse the nested SES event from the Message field const body = JSON.parse(req.body.Message); + const claimResult = await claimSnsNotification(snsMessageId); + if (claimResult.outcome === 'completed') { + return res.status(200).json({success: true, duplicate: true}); + } + if (claimResult.outcome === 'in-flight') { + return res.status(503).json({success: false, message: 'SNS notification is already being processed'}); + } + activeSnsClaim = claimResult.claim; + stopSnsClaimHeartbeat = startSnsClaimHeartbeat(activeSnsClaim); + // Check if this is an inbound email notification (SES Receiving) if (body.notificationType === 'Received') { signale.info('[WEBHOOK] Received inbound email notification from SES'); try { - // Extract recipient addresses from the inbound email const recipients = body.receipt?.recipients || []; if (recipients.length === 0) { signale.warn('[WEBHOOK] No recipients found in inbound email'); + await completeActiveClaim(); return res.status(200).json({success: true, message: 'No recipients found'}); } - // For each recipient, identify the domain and create events + const senderEmail = body.mail?.source; + if (typeof senderEmail !== 'string' || senderEmail.length === 0) { + throw new Error('Inbound SNS notification is missing mail.source'); + } + const normalizedSender = ContactService.normalizeEmail(senderEmail); + const senderFromHeader = body.mail?.commonHeaders?.from?.[0] || senderEmail; + let htmlBody: string | undefined; + + if (body.content && typeof body.content === 'string') { + try { + const isBase64 = body.receipt?.action?.encoding === 'BASE64'; + const emailBuffer = isBase64 ? Buffer.from(body.content, 'base64') : Buffer.from(body.content); + const parsed = await simpleParser(emailBuffer); + const raw = + (parsed.html ? String(parsed.html) : undefined) ?? parsed.textAsHtml ?? parsed.text ?? undefined; + + if (raw) { + htmlBody = sanitizeHtml(raw, { + allowedTags: sanitizeHtml.defaults.allowedTags.concat(['img']), + allowedAttributes: { + ...sanitizeHtml.defaults.allowedAttributes, + img: ['src', 'alt', 'width', 'height'], + '*': ['style'], + }, + allowedSchemes: ['http', 'https', 'mailto'], + }); + } + } catch (parseError) { + signale.error('[WEBHOOK] Failed to parse email content:', parseError); + } + } + + const targets: Array<{ + recipientEmail: string; + project: {id: string; name: string; customer: string | null}; + }> = []; + for (const recipient of recipients) { const recipientEmail = recipient as string; const domain = recipientEmail.split('@')[1]; @@ -152,140 +368,132 @@ export class Webhooks { continue; } - signale.info( - `[WEBHOOK] Found ${domainRecords.length} project(s) with verified domain ${domain}. Processing inbound email for all.`, - ); - - // Extract sender information (same for all projects) - const senderEmail = body.mail?.source; - const senderFromHeader = body.mail?.commonHeaders?.from?.[0] || senderEmail; - - // Parse email content if available - let htmlBody: string | undefined; - - if (body.content && typeof body.content === 'string') { - try { - const isBase64 = body.receipt?.action?.encoding === 'BASE64'; - const emailBuffer = isBase64 - ? Buffer.from(body.content, 'base64') - : Buffer.from(body.content); - - const parsed = await simpleParser(emailBuffer); - const raw = - (parsed.html ? String(parsed.html) : undefined) ?? - parsed.textAsHtml ?? - parsed.text ?? - undefined; - - if (raw) { - htmlBody = sanitizeHtml(raw, { - allowedTags: sanitizeHtml.defaults.allowedTags.concat(['img']), - allowedAttributes: { - ...sanitizeHtml.defaults.allowedAttributes, - img: ['src', 'alt', 'width', 'height'], - '*': ['style'], - }, - allowedSchemes: ['http', 'https', 'mailto'], - }); - } - } catch (parseError) { - signale.error('[WEBHOOK] Failed to parse email content:', parseError); - } - } - - // Process inbound email for each project that has this domain verified for (const domainRecord of domainRecords) { signale.info(`[WEBHOOK] Processing inbound email for project: ${domainRecord.project.name}`); - - // Check billing limits before processing inbound email const limitCheck = await BillingLimitService.checkLimit(domainRecord.projectId, EmailSourceType.INBOUND); if (!limitCheck.allowed) { signale.warn( `[WEBHOOK] Inbound email blocked for project ${domainRecord.project.name}: ${limitCheck.message}`, ); - continue; // Skip this project but continue processing for other projects + continue; } - // Find or create a contact for the sender in this project - let contact; - if (senderEmail) { - contact = await ContactService.upsert( - domainRecord.projectId, - senderEmail, - undefined, // No additional data - true, // Subscribe by default for inbound email senders - ); + targets.push({recipientEmail, project: domainRecord.project}); + } + } + + const claim = activeSnsClaim; + if (!claim) { + throw new Error(`SNS notification ${snsMessageId} lost its processing claim`); + } + + const committed = await prisma.$transaction(async tx => { + const effects: Array<{ + emailId: string; + eventIds: string[]; + project: {id: string; name: string; customer: string | null}; + recipientEmail: string; + }> = []; + + for (const target of targets) { + const existingContact = await tx.contact.findUnique({ + where: {projectId_email: {projectId: target.project.id, email: normalizedSender}}, + }); + const contact = existingContact + ? await tx.contact.update({ + where: {id: existingContact.id}, + data: {subscribed: true}, + }) + : await tx.contact.create({ + data: {projectId: target.project.id, email: normalizedSender, subscribed: true}, + }); + const eventIds: string[] = []; + + if (existingContact && !existingContact.subscribed) { + const subscribedEvent = await tx.event.create({ + data: { + projectId: target.project.id, + contactId: contact.id, + name: 'contact.subscribed', + }, + }); + eventIds.push(subscribedEvent.id); } - // Create an Email record for tracking with parsed content - const inboundEmail = await prisma.email.create({ + const inboundEmail = await tx.email.create({ data: { - projectId: domainRecord.projectId, - contactId: contact!.id, + projectId: target.project.id, + contactId: contact.id, subject: body.mail?.commonHeaders?.subject || '(No subject)', - body: htmlBody || '', // Store HTML body in the body field - from: recipientEmail, // The recipient address that received the email + body: htmlBody || '', + from: target.recipientEmail, sourceType: EmailSourceType.INBOUND, - status: EmailStatus.RECEIVED, // Inbound emails use RECEIVED status + status: EmailStatus.RECEIVED, deliveredAt: new Date(body.mail?.timestamp || new Date()), }, }); - - // Increment usage counter in cache - await BillingLimitService.incrementUsage(domainRecord.projectId, EmailSourceType.INBOUND); - - // Record Stripe metering if project has customer - if (domainRecord.project.customer) { - await MeterService.recordEmailSent( - domainRecord.project.customer, - 1, // Inbound emails count as 1 credit - `email_${inboundEmail.id}`, - ); - } - - // Prepare event data with all inbound email details including body content const eventData = { messageId: body.mail?.messageId, from: senderEmail, fromHeader: senderFromHeader, - to: recipientEmail, + to: target.recipientEmail, subject: body.mail?.commonHeaders?.subject, timestamp: body.mail?.timestamp, recipients: body.receipt?.recipients, hasContent: !!body.content, - // Email body content body: htmlBody, - // Security verdicts spamVerdict: body.receipt?.spamVerdict?.status, virusVerdict: body.receipt?.virusVerdict?.status, spfVerdict: body.receipt?.spfVerdict?.status, dkimVerdict: body.receipt?.dkimVerdict?.status, dmarcVerdict: body.receipt?.dmarcVerdict?.status, - // Processing metadata processingTimeMillis: body.receipt?.processingTimeMillis, }; + const receivedEvent = await tx.event.create({ + data: { + projectId: target.project.id, + contactId: contact.id, + emailId: inboundEmail.id, + name: 'email.received', + data: toPrismaJson(eventData), + }, + }); + eventIds.push(receivedEvent.id); + effects.push({ + emailId: inboundEmail.id, + eventIds, + project: target.project, + recipientEmail: target.recipientEmail, + }); + } - // Create the email.received event (this will trigger workflows) - await EventService.trackEvent( - domainRecord.projectId, - 'email.received', - contact?.id, - inboundEmail.id, // Link the event to the inbound email record - eventData, - ); + await completeSnsNotification(claim, tx); + return effects; + }); + activeSnsClaim = undefined; - signale.success( - `[WEBHOOK] Created email.received event for ${senderEmail} → ${recipientEmail} (project: ${domainRecord.project.name})`, - ); + for (const effect of committed) { + await BillingLimitService.incrementUsage(effect.project.id, EmailSourceType.INBOUND); + if (effect.project.customer) { + await MeterService.recordEmailSent(effect.project.customer, 1, `email_${effect.emailId}`); + } + for (const eventId of effect.eventIds) { + try { + await EventService.dispatchStoredEvent(eventId); + } catch (dispatchError) { + signale.error(`[WEBHOOK] Deferred workflow dispatch for event ${eventId}:`, dispatchError); + } } + signale.success( + `[WEBHOOK] Created email.received event for ${senderEmail} → ${effect.recipientEmail} (project: ${effect.project.name})`, + ); } return res.status(200).json({success: true, message: 'Inbound email processed'}); } catch (inboundError) { signale.error('[WEBHOOK] Error processing inbound email:', inboundError); - // Return 200 to acknowledge receipt even if processing failed - return res.status(200).json({success: true, message: 'Error processing inbound email'}); + throw inboundError; } } @@ -295,6 +503,7 @@ export class Webhooks { if (!messageId) { signale.warn('[WEBHOOK] No messageId found in SNS notification'); + await completeActiveClaim(); return res.status(400).json({success: false, error: 'No messageId found'}); } @@ -309,15 +518,22 @@ export class Webhooks { if (!email) { // Error level for the same reason as a signature failure: an event that matches no - // email row is silently lost, and SES gives up after its retries. A run of these - // means the send path is not stamping `messageId`, which is invisible from outside. - signale.error(`[WEBHOOK] ${eventType} event dropped — no email found for messageId: ${messageId}`); - return res.status(404).json({success: false, error: 'Email not found'}); + // email row is silently lost. A run of these means the send path is not stamping + // `messageId`, which is invisible from outside. + signale.error(`[WEBHOOK] ${eventType} event has no email for messageId: ${messageId}`); + // SES can publish before the sender has persisted its returned messageId. + // Keep the receipt retryable so that race cannot permanently lose the event. + throw new Error(`Email not found for messageId: ${messageId}`); } const now = new Date(); const updateData: Prisma.EmailUpdateInput = {}; const eventName = `email.${eventType.toLowerCase()}`; + let unsubscribeContact = false; + let bounceNotification = false; + let bounceNotificationType: string | undefined; + let complaintNotification = false; + let enforceSecurityLimits = false; // Base event data with email metadata const baseEventData = { @@ -350,14 +566,8 @@ export class Webhooks { if (!email.openedAt) { updateData.openedAt = now; } - updateData.opens = (email.opens || 0) + 1; + updateData.opens = {increment: 1}; updateData.status = EmailStatus.OPENED; - eventData = { - ...baseEventData, - openedAt: email.openedAt?.toISOString() || now.toISOString(), - opens: (email.opens || 0) + 1, - isFirstOpen: !email.openedAt, - }; break; case 'Click': { @@ -367,14 +577,11 @@ export class Webhooks { if (!email.clickedAt) { updateData.clickedAt = now; } - updateData.clicks = (email.clicks || 0) + 1; + updateData.clicks = {increment: 1}; updateData.status = EmailStatus.CLICKED; eventData = { ...baseEventData, link: clickedLink, - clickedAt: email.clickedAt?.toISOString() || now.toISOString(), - clicks: (email.clicks || 0) + 1, - isFirstClick: !email.clickedAt, }; break; } @@ -389,19 +596,15 @@ export class Webhooks { signale.warn(`[WEBHOOK] Permanent bounce received for ${email.contact.email} from ${email.project.name}`); updateData.status = EmailStatus.BOUNCED; updateData.bouncedAt = now; - // Unsubscribe contact on permanent bounce - await prisma.contact.update({ - where: {id: email.contactId}, - data: {subscribed: false}, - }); + unsubscribeContact = true; + bounceNotification = true; + bounceNotificationType = bounceType; + enforceSecurityLimits = true; eventData = { ...baseEventData, bounceType, bouncedAt: now.toISOString(), }; - - // Send notification about permanent bounce - await NtfyService.notifyEmailBounce(email.project.name, email.projectId, email.contact.email, bounceType); } else if (isTransientBounce) { // Soft bounce (e.g., out-of-office, mailbox full) - don't count toward bounce rate signale.info( @@ -421,17 +624,14 @@ export class Webhooks { ); updateData.status = EmailStatus.BOUNCED; updateData.bouncedAt = now; - await prisma.contact.update({ - where: {id: email.contactId}, - data: {subscribed: false}, - }); + unsubscribeContact = true; + bounceNotification = true; + bounceNotificationType = bounceType; eventData = { ...baseEventData, bounceType, bouncedAt: now.toISOString(), }; - - await NtfyService.notifyEmailBounce(email.project.name, email.projectId, email.contact.email, bounceType); } break; } @@ -440,30 +640,71 @@ export class Webhooks { signale.warn(`[WEBHOOK] Complaint received for ${email.contact.email} from ${email.project.name}`); updateData.status = EmailStatus.COMPLAINED; updateData.complainedAt = now; - // Unsubscribe contact on complaint - await prisma.contact.update({ - where: {id: email.contactId}, - data: {subscribed: false}, - }); + unsubscribeContact = true; + complaintNotification = true; + enforceSecurityLimits = true; eventData = { ...baseEventData, complainedAt: now.toISOString(), }; - - // Send notification about complaint - await NtfyService.notifyEmailComplaint(email.project.name, email.projectId, email.contact.email); break; default: signale.warn(`[WEBHOOK] Unknown event type: ${eventType}`); + await completeActiveClaim(); return res.status(200).json({success: true}); } - // Update email with new status and timestamps - await prisma.email.update({ - where: {id: email.id}, - data: updateData, + const claim = activeSnsClaim; + if (!claim) { + throw new Error(`SNS notification ${snsMessageId} lost its processing claim`); + } + + // The business mutation, durable event, and receipt completion share one + // commit. A database failure therefore leaves no partial effects for the + // SNS retry to duplicate. + const storedEvent = await prisma.$transaction(async tx => { + if (unsubscribeContact) { + await tx.contact.update({ + where: {id: email.contactId}, + data: {subscribed: false}, + }); + } + + const updatedEmail = await tx.email.update({ + where: {id: email.id}, + data: updateData, + }); + + if (eventType === 'Open') { + eventData = { + ...baseEventData, + openedAt: updatedEmail.openedAt?.toISOString(), + opens: updatedEmail.opens, + isFirstOpen: !email.openedAt, + }; + } else if (eventType === 'Click') { + eventData = { + ...eventData, + clickedAt: updatedEmail.clickedAt?.toISOString(), + clicks: updatedEmail.clicks, + isFirstClick: !email.clickedAt, + }; + } + + const event = await tx.event.create({ + data: { + projectId: email.projectId, + contactId: email.contactId, + emailId: email.id, + name: eventName, + data: toPrismaJson(eventData), + }, + }); + await completeSnsNotification(claim, tx); + return event; }); + activeSnsClaim = undefined; // The campaign counters the stats endpoint reads live on the campaign row, and this // event has just moved one of them. They are not incremented from here: this handler @@ -474,13 +715,30 @@ export class Webhooks { await CampaignService.markStatsDirty(email.campaignId); } - // Track event (this will trigger workflows) - await EventService.trackEvent(email.projectId, eventName, email.contactId, email.id, eventData); + if (bounceNotification) { + try { + await NtfyService.notifyEmailBounce( + email.project.name, + email.projectId, + email.contact.email, + bounceNotificationType, + ); + } catch (notificationError) { + signale.error('[WEBHOOK] Failed to notify about email bounce:', notificationError); + } + } else if (complaintNotification) { + await NtfyService.notifyEmailComplaint(email.project.name, email.projectId, email.contact.email); + } + + try { + await EventService.dispatchStoredEvent(storedEvent.id); + } catch (dispatchError) { + // The event row is the outbox. The maintenance worker can retry a null + // processedAt without asking SNS to replay committed email effects. + signale.error(`[WEBHOOK] Deferred workflow dispatch for event ${storedEvent.id}:`, dispatchError); + } - // Check security limits only for permanent bounces and complaints - // Transient bounces (soft bounces) don't count toward bounce rate - const isPermanentBounce = eventType === 'Bounce' && body.bounce?.bounceType === 'Permanent'; - if (isPermanentBounce || eventType === 'Complaint') { + if (enforceSecurityLimits) { await SecurityService.checkAndEnforceSecurityLimits(email.projectId); } @@ -488,8 +746,18 @@ export class Webhooks { return res.status(200).json({success: true}); } catch (error) { signale.error('[WEBHOOK] Error processing SNS webhook:', error); - // Always return 200 to prevent SNS from retrying - return res.status(200).json({success: true}); + if (activeSnsClaim) { + try { + await failSnsNotification(activeSnsClaim); + } catch (settleError) { + // The processing lease is the fallback if the database is unavailable + // while recording failure. The delivery still receives 5xx and retries. + signale.error(`[WEBHOOK] Failed to release SNS claim ${activeSnsClaim.messageId}:`, settleError); + } + } + return res.status(500).json({success: false, message: 'Failed to process SNS notification'}); + } finally { + stopSnsClaimHeartbeat?.(); } } diff --git a/apps/api/src/controllers/Workflows.ts b/apps/api/src/controllers/Workflows.ts index 734ceebbd..f417701f3 100644 --- a/apps/api/src/controllers/Workflows.ts +++ b/apps/api/src/controllers/Workflows.ts @@ -311,7 +311,7 @@ export class Workflows { public async createTransition(req: Request, res: Response, _next: NextFunction) { const auth = res.locals.auth; const workflowId = req.params.id; - const {fromStepId, toStepId, condition, priority} = req.body; + const {fromStepId, toStepId, condition, waitOutcome, priority} = req.body; if (!workflowId) { return res.status(400).json({error: 'Workflow ID is required'}); @@ -325,6 +325,7 @@ export class Workflows { fromStepId, toStepId, condition, + waitOutcome, priority, }); diff --git a/apps/api/src/controllers/__tests__/Actions.track-durability.test.ts b/apps/api/src/controllers/__tests__/Actions.track-durability.test.ts new file mode 100644 index 000000000..8681392d0 --- /dev/null +++ b/apps/api/src/controllers/__tests__/Actions.track-durability.test.ts @@ -0,0 +1,679 @@ +import type {NextFunction, Request, Response} from 'express'; +import express from 'express'; +import type {IdempotencyKeyCleanupJobData} from '@plunk/types'; +import type {Job} from 'bullmq'; +import request from 'supertest'; +import {beforeEach, describe, expect, it, vi} from 'vitest'; + +import {factories, getPrismaClient} from '../../../../../test/helpers'; +import {prisma as servicePrisma} from '../../database/prisma.js'; +import {processCleanup} from '../../jobs/idempotency-key-cleanup-processor.js'; +import {EventService} from '../../services/EventService.js'; +import {QueueService} from '../../services/QueueService.js'; +import {Actions} from '../Actions.js'; + +function createTrackApp(projectId: string) { + const app = express(); + const actions = new Actions(); + + app.use(express.json()); + app.post('/v1/track', (req, res, next) => { + res.locals.auth = {projectId}; + void actions.track(req, res, next); + }); + app.use((error: Error, _req: Request, res: Response, _next: NextFunction) => { + res.status(500).json({success: false, error: error.message}); + }); + + return app; +} + +describe('POST /v1/track durability', () => { + const prisma = getPrismaClient(); + let projectId: string; + + beforeEach(async () => { + const {project} = await factories.createUserWithProject(); + projectId = project.id; + }); + + it('returns the stored event after synchronous dispatch succeeds', async () => { + const response = await request(createTrackApp(projectId)) + .post('/v1/track') + .send({ + event: 'checkout.completed', + email: 'success@example.com', + data: {orderId: 'order-123'}, + }); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ + success: true, + data: { + contact: expect.any(String), + event: expect.any(String), + timestamp: expect.any(String), + }, + }); + expect((await prisma.event.findUniqueOrThrow({where: {id: response.body.data.event}})).processedAt).not.toBeNull(); + }); + + it('acknowledges a stored event when dispatch fails, then reconciles it once', async () => { + const workflow = await factories.createWorkflow({ + projectId, + enabled: true, + allowReentry: false, + triggerType: 'EVENT', + triggerConfig: {eventName: 'invoice.paid'}, + }); + const workflowLookup = vi + .spyOn(servicePrisma.workflow, 'findUnique') + .mockRejectedValueOnce(new Error('injected workflow start failure')); + + const response = await request(createTrackApp(projectId)) + .post('/v1/track') + .send({ + event: 'invoice.paid', + email: 'recovery@example.com', + data: {invoiceId: 'invoice-123'}, + }); + + expect(response.status).toBe(200); + expect(response.body.data.event).toEqual(expect.any(String)); + const pendingEvent = await prisma.event.findFirstOrThrow({ + where: {projectId, name: 'invoice.paid'}, + }); + expect(pendingEvent.processedAt).toBeNull(); + expect(pendingEvent.dispatchAttempts).toBe(1); + expect(await prisma.workflowExecution.count({where: {workflowId: workflow.id}})).toBe(0); + + workflowLookup.mockRestore(); + await prisma.event.update({ + where: {id: pendingEvent.id}, + data: { + createdAt: new Date(Date.now() - 6 * 60 * 1000), + nextDispatchAt: new Date(Date.now() - 1000), + }, + }); + const updateProgress = vi.fn().mockResolvedValue(undefined); + const job = {updateProgress} as unknown as Job; + + await processCleanup(job); + await processCleanup(job); + + expect((await prisma.event.findUniqueOrThrow({where: {id: pendingEvent.id}})).processedAt).not.toBeNull(); + expect(await prisma.workflowExecution.count({where: {workflowId: workflow.id}})).toBe(1); + expect(updateProgress).toHaveBeenCalledWith(100); + }); + + it('does not replay a failed step after the event enrolled its execution', async () => { + const workflow = await factories.createWorkflow({ + projectId, + enabled: true, + allowReentry: false, + triggerType: 'EVENT', + triggerConfig: {eventName: 'shipment.created'}, + }); + const completeStep = vi + .spyOn(servicePrisma.workflowStepExecution, 'update') + .mockRejectedValueOnce(new Error('injected failure after workflow enrollment')); + + const response = await request(createTrackApp(projectId)) + .post('/v1/track') + .send({ + event: 'shipment.created', + email: 'resume@example.com', + data: {shipmentId: 'shipment-123'}, + }); + + expect(response.status).toBe(200); + const event = await prisma.event.findUniqueOrThrow({where: {id: response.body.data.event}}); + const failedExecution = await prisma.workflowExecution.findFirstOrThrow({ + where: {workflowId: workflow.id, sourceEventId: event.id}, + }); + expect(event.processedAt).toBeNull(); + expect(failedExecution.status).toBe('FAILED'); + + completeStep.mockRestore(); + await prisma.event.update({ + where: {id: event.id}, + data: { + createdAt: new Date(Date.now() - 6 * 60 * 1000), + nextDispatchAt: new Date(Date.now() - 1000), + }, + }); + const updateProgress = vi.fn().mockResolvedValue(undefined); + const job = {updateProgress} as unknown as Job; + + await processCleanup(job); + await processCleanup(job); + + expect((await prisma.event.findUniqueOrThrow({where: {id: event.id}})).processedAt).not.toBeNull(); + expect(await prisma.workflowExecution.count({where: {workflowId: workflow.id}})).toBe(1); + expect( + await prisma.workflowStepExecution.count({ + where: {executionId: failedExecution.id}, + }), + ).toBe(1); + expect((await prisma.workflowExecution.findUniqueOrThrow({where: {id: failedExecution.id}})).status).toBe('FAILED'); + }); + + it('resumes an event execution that failed before its first step began', async () => { + const workflow = await factories.createWorkflow({ + projectId, + enabled: true, + allowReentry: false, + triggerType: 'EVENT', + triggerConfig: {eventName: 'execution.created'}, + }); + const executionLookup = vi + .spyOn(servicePrisma.workflowExecution, 'findUnique') + .mockRejectedValueOnce(new Error('injected failure before trigger step')); + + const response = await request(createTrackApp(projectId)) + .post('/v1/track') + .send({event: 'execution.created', email: 'unstarted@example.com'}); + + expect(response.status).toBe(200); + const event = await prisma.event.findUniqueOrThrow({where: {id: response.body.data.event}}); + const unstartedExecution = await prisma.workflowExecution.findFirstOrThrow({ + where: {workflowId: workflow.id, sourceEventId: event.id}, + }); + expect(event.processedAt).toBeNull(); + expect(unstartedExecution.status).toBe('RUNNING'); + expect(await prisma.workflowStepExecution.count({where: {executionId: unstartedExecution.id}})).toBe(0); + + executionLookup.mockRestore(); + await prisma.event.update({ + where: {id: event.id}, + data: { + createdAt: new Date(Date.now() - 6 * 60 * 1000), + nextDispatchAt: new Date(Date.now() - 1000), + }, + }); + const updateProgress = vi.fn().mockResolvedValue(undefined); + + await processCleanup({updateProgress} as unknown as Job); + + expect((await prisma.event.findUniqueOrThrow({where: {id: event.id}})).processedAt).not.toBeNull(); + expect(await prisma.workflowExecution.count({where: {workflowId: workflow.id, sourceEventId: event.id}})).toBe(1); + expect(await prisma.workflowStepExecution.count({where: {executionId: unstartedExecution.id}})).toBe(1); + expect((await prisma.workflowExecution.findUniqueOrThrow({where: {id: unstartedExecution.id}})).status).toBe( + 'COMPLETED', + ); + }); + + it('retries a wait continuation claimed before a transient queue failure', async () => { + const workflow = await factories.createWorkflow({ + projectId, + enabled: true, + triggerType: 'EVENT', + triggerConfig: {eventName: 'unrelated.event'}, + }); + const waitStep = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: 'WAIT_FOR_EVENT', + name: 'Wait for payment', + position: {x: 100, y: 0}, + config: {eventName: 'payment.settled', timeout: 3600}, + }, + }); + const exitStep = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: 'EXIT', + name: 'Paid', + position: {x: 200, y: 0}, + config: {}, + }, + }); + await prisma.workflowTransition.create({ + data: {fromStepId: waitStep.id, toStepId: exitStep.id, waitOutcome: 'EVENT'}, + }); + const contact = await factories.createContact({projectId}); + const execution = await prisma.workflowExecution.create({ + data: { + workflowId: workflow.id, + contactId: contact.id, + status: 'WAITING', + currentStepId: waitStep.id, + context: {source: 'signup'}, + }, + }); + const waitExecution = await prisma.workflowStepExecution.create({ + data: { + executionId: execution.id, + stepId: waitStep.id, + status: 'WAITING', + startedAt: new Date(), + }, + }); + const cancelTimeout = vi + .spyOn(QueueService, 'cancelWorkflowTimeout') + .mockRejectedValueOnce(new Error('injected Redis failure')); + + const response = await request(createTrackApp(projectId)) + .post('/v1/track') + .send({event: 'payment.settled', email: contact.email, data: {invoiceId: 'invoice-456'}}); + + expect(response.status).toBe(200); + const event = await prisma.event.findUniqueOrThrow({where: {id: response.body.data.event}}); + expect(event.processedAt).toBeNull(); + expect( + await prisma.workflowStepExecution.findUniqueOrThrow({ + where: {id: waitExecution.id}, + select: {status: true, resumeEventId: true, output: true}, + }), + ).toMatchObject({ + status: 'RUNNING', + resumeEventId: event.id, + output: {eventData: {invoiceId: 'invoice-456'}}, + }); + + cancelTimeout.mockRestore(); + await prisma.event.update({ + where: {id: event.id}, + data: { + createdAt: new Date(Date.now() - 6 * 60 * 1000), + nextDispatchAt: new Date(Date.now() - 1000), + }, + }); + const updateProgress = vi.fn().mockResolvedValue(undefined); + const job = {updateProgress} as unknown as Job; + + await processCleanup(job); + await processCleanup(job); + + expect((await prisma.event.findUniqueOrThrow({where: {id: event.id}})).processedAt).not.toBeNull(); + expect( + await prisma.workflowStepExecution.findUniqueOrThrow({ + where: {id: waitExecution.id}, + select: {status: true, resumeEventId: true}, + }), + ).toEqual({status: 'COMPLETED', resumeEventId: event.id}); + expect(await prisma.workflowStepExecution.count({where: {executionId: execution.id, stepId: exitStep.id}})).toBe(1); + }); + + it('does not replay a delivered wait continuation when its acknowledgement fails', async () => { + const workflow = await factories.createWorkflow({ + projectId, + enabled: true, + triggerType: 'EVENT', + triggerConfig: {eventName: 'unrelated.event'}, + }); + const waitStep = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: 'WAIT_FOR_EVENT', + name: 'Wait for capture', + position: {x: 100, y: 0}, + config: {eventName: 'payment.captured', timeout: 3600}, + }, + }); + const exitStep = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: 'EXIT', + name: 'Captured', + position: {x: 200, y: 0}, + config: {}, + }, + }); + await prisma.workflowTransition.create({ + data: {fromStepId: waitStep.id, toStepId: exitStep.id, waitOutcome: 'EVENT'}, + }); + const contact = await factories.createContact({projectId}); + const execution = await prisma.workflowExecution.create({ + data: { + workflowId: workflow.id, + contactId: contact.id, + status: 'WAITING', + currentStepId: waitStep.id, + }, + }); + const waitExecution = await prisma.workflowStepExecution.create({ + data: { + executionId: execution.id, + stepId: waitStep.id, + status: 'WAITING', + startedAt: new Date(), + }, + }); + const updateWait = servicePrisma.workflowStepExecution.updateMany.bind(servicePrisma.workflowStepExecution); + let failAcknowledgement = true; + const waitUpdate = vi.spyOn(servicePrisma.workflowStepExecution, 'updateMany').mockImplementation(async args => { + if (failAcknowledgement && args.where?.id === waitExecution.id && args.data.status === 'COMPLETED') { + failAcknowledgement = false; + throw new Error('injected acknowledgement failure'); + } + return updateWait(args); + }); + + const response = await request(createTrackApp(projectId)) + .post('/v1/track') + .send({event: 'payment.captured', email: contact.email}); + + expect(response.status).toBe(200); + const event = await prisma.event.findUniqueOrThrow({where: {id: response.body.data.event}}); + expect(event.processedAt).toBeNull(); + expect(await prisma.workflowStepExecution.count({where: {executionId: execution.id, stepId: exitStep.id}})).toBe(1); + expect((await prisma.workflowStepExecution.findUniqueOrThrow({where: {id: waitExecution.id}})).status).toBe( + 'RUNNING', + ); + + waitUpdate.mockRestore(); + await prisma.event.update({ + where: {id: event.id}, + data: { + createdAt: new Date(Date.now() - 6 * 60 * 1000), + nextDispatchAt: new Date(Date.now() - 1000), + }, + }); + const updateProgress = vi.fn().mockResolvedValue(undefined); + const job = {updateProgress} as unknown as Job; + + await processCleanup(job); + await processCleanup(job); + + expect((await prisma.event.findUniqueOrThrow({where: {id: event.id}})).processedAt).not.toBeNull(); + expect((await prisma.workflowStepExecution.findUniqueOrThrow({where: {id: waitExecution.id}})).status).toBe( + 'COMPLETED', + ); + expect(await prisma.workflowStepExecution.count({where: {executionId: execution.id, stepId: exitStep.id}})).toBe(1); + }); + + it('continues enrolling independent workflows when one target fails', async () => { + const firstWorkflow = await factories.createWorkflow({ + projectId, + enabled: true, + allowReentry: true, + triggerType: 'EVENT', + triggerConfig: {eventName: 'account.ready'}, + }); + const secondWorkflow = await factories.createWorkflow({ + projectId, + enabled: true, + allowReentry: true, + triggerType: 'EVENT', + triggerConfig: {eventName: 'account.ready'}, + }); + const workflowLookup = vi + .spyOn(servicePrisma.workflow, 'findUnique') + .mockRejectedValueOnce(new Error('injected failure for one workflow')); + + const response = await request(createTrackApp(projectId)) + .post('/v1/track') + .send({event: 'account.ready', email: 'fanout@example.com'}); + + expect(response.status).toBe(200); + expect( + await prisma.workflowExecution.count({ + where: {workflowId: {in: [firstWorkflow.id, secondWorkflow.id]}}, + }), + ).toBe(1); + expect( + await prisma.event.findUniqueOrThrow({ + where: {id: response.body.data.event}, + select: {processedAt: true, dispatchAttempts: true}, + }), + ).toEqual({processedAt: null, dispatchAttempts: 1}); + + workflowLookup.mockRestore(); + }); + + it('enrolls an event in a workflow once under concurrent dispatch', async () => { + const workflow = await factories.createWorkflow({ + projectId, + enabled: true, + allowReentry: true, + triggerType: 'EVENT', + triggerConfig: {eventName: 'concurrent.event'}, + }); + const contact = await factories.createContact({projectId}); + const event = await prisma.event.create({ + data: { + projectId, + contactId: contact.id, + name: 'concurrent.event', + }, + }); + + const originalFindFirst = servicePrisma.workflowExecution.findFirst.bind(servicePrisma.workflowExecution); + let sourceLookups = 0; + let releaseSourceLookups!: () => void; + const bothAtSourceLookup = new Promise(resolve => { + releaseSourceLookups = resolve; + }); + const executionLookup = vi.spyOn(servicePrisma.workflowExecution, 'findFirst').mockImplementation(async args => { + if (args.where?.workflowId === workflow.id && args.where?.sourceEventId === event.id) { + sourceLookups += 1; + if (sourceLookups === 2) releaseSourceLookups(); + await bothAtSourceLookup; + return null; + } + + if ( + args.where?.workflowId === workflow.id && + args.where?.contactId === contact.id && + args.where?.status === 'RUNNING' + ) { + return null; + } + + return originalFindFirst(args); + }); + const startWorkflowForContact = ( + EventService as unknown as { + startWorkflowForContact( + workflowId: string, + contactId: string, + sourceEventId: string, + context?: Record, + ): Promise; + } + ).startWorkflowForContact.bind(EventService); + + await Promise.all([ + startWorkflowForContact(workflow.id, contact.id, event.id), + startWorkflowForContact(workflow.id, contact.id, event.id), + ]); + + executionLookup.mockRestore(); + expect(await prisma.workflowExecution.count({where: {workflowId: workflow.id, sourceEventId: event.id}})).toBe(1); + }); + + it('does not let reconciliation overlap an active event dispatch lease', async () => { + const event = await prisma.event.create({ + data: { + projectId, + name: 'slow.dispatch', + createdAt: new Date(Date.now() - 6 * 60 * 1000), + }, + }); + let releaseDispatch!: () => void; + const dispatchReleased = new Promise(resolve => { + releaseDispatch = resolve; + }); + let markDispatchStarted!: () => void; + const dispatchStarted = new Promise(resolve => { + markDispatchStarted = resolve; + }); + const eventService = EventService as unknown as { + triggerWorkflows(...args: unknown[]): Promise; + }; + const triggerWorkflows = vi.spyOn(eventService, 'triggerWorkflows').mockImplementation(async () => { + markDispatchStarted(); + await dispatchReleased; + return []; + }); + + const initialDispatch = EventService.dispatchStoredEvent(event.id); + await dispatchStarted; + expect( + await prisma.event.findUniqueOrThrow({ + where: {id: event.id}, + select: {dispatchLeaseId: true, dispatchLeaseExpiresAt: true}, + }), + ).toMatchObject({ + dispatchLeaseId: expect.any(String), + dispatchLeaseExpiresAt: expect.any(Date), + }); + + const updateProgress = vi.fn().mockResolvedValue(undefined); + await processCleanup({updateProgress} as unknown as Job); + + expect(triggerWorkflows).toHaveBeenCalledTimes(1); + expect((await prisma.event.findUniqueOrThrow({where: {id: event.id}})).processedAt).toBeNull(); + + releaseDispatch(); + await initialDispatch; + triggerWorkflows.mockRestore(); + + expect( + await prisma.event.findUniqueOrThrow({ + where: {id: event.id}, + select: {processedAt: true, dispatchLeaseId: true, dispatchLeaseExpiresAt: true}, + }), + ).toMatchObject({ + processedAt: expect.any(Date), + dispatchLeaseId: null, + dispatchLeaseExpiresAt: null, + }); + }); + + it('renews an active dispatch lease while a long workflow target is running', async () => { + const event = await prisma.event.create({ + data: { + projectId, + name: 'slow.heartbeat', + createdAt: new Date(Date.now() - 6 * 60 * 1000), + }, + }); + const eventService = EventService as unknown as { + startDispatchLeaseHeartbeat( + eventId: string, + leaseId: string, + intervalMs?: number, + ): {assertOwnership: () => Promise; stop: () => void}; + triggerWorkflows(...args: unknown[]): Promise; + }; + const originalHeartbeat = eventService.startDispatchLeaseHeartbeat.bind(EventService); + const heartbeat = vi + .spyOn(eventService, 'startDispatchLeaseHeartbeat') + .mockImplementation((eventId, leaseId) => originalHeartbeat(eventId, leaseId, 20)); + let releaseDispatch!: () => void; + const dispatchReleased = new Promise(resolve => { + releaseDispatch = resolve; + }); + let markDispatchStarted!: () => void; + const dispatchStarted = new Promise(resolve => { + markDispatchStarted = resolve; + }); + const triggerWorkflows = vi.spyOn(eventService, 'triggerWorkflows').mockImplementation(async () => { + markDispatchStarted(); + await dispatchReleased; + return []; + }); + + const initialDispatch = EventService.dispatchStoredEvent(event.id); + await dispatchStarted; + const activeLease = await prisma.event.findUniqueOrThrow({where: {id: event.id}}); + await prisma.event.update({ + where: {id: event.id}, + data: {dispatchLeaseExpiresAt: new Date(Date.now() + 5)}, + }); + await new Promise(resolve => setTimeout(resolve, 60)); + + const renewedLease = await prisma.event.findUniqueOrThrow({where: {id: event.id}}); + expect(renewedLease.dispatchLeaseId).toBe(activeLease.dispatchLeaseId); + expect(renewedLease.dispatchLeaseExpiresAt!.getTime()).toBeGreaterThan(Date.now()); + + const updateProgress = vi.fn().mockResolvedValue(undefined); + await processCleanup({updateProgress} as unknown as Job); + expect(triggerWorkflows).toHaveBeenCalledTimes(1); + + releaseDispatch(); + await initialDispatch; + triggerWorkflows.mockRestore(); + heartbeat.mockRestore(); + }); + + it('reclaims an expired event dispatch lease', async () => { + const event = await prisma.event.create({ + data: { + projectId, + name: 'crashed.dispatch', + createdAt: new Date(Date.now() - 20 * 60 * 1000), + dispatchLeaseId: 'expired-lease', + dispatchLeaseExpiresAt: new Date(Date.now() - 60 * 1000), + }, + }); + const updateProgress = vi.fn().mockResolvedValue(undefined); + + await processCleanup({updateProgress} as unknown as Job); + + expect( + await prisma.event.findUniqueOrThrow({ + where: {id: event.id}, + select: {processedAt: true, dispatchLeaseId: true, dispatchLeaseExpiresAt: true}, + }), + ).toMatchObject({ + processedAt: expect.any(Date), + dispatchLeaseId: null, + dispatchLeaseExpiresAt: null, + }); + }); + + it('dead-letters a poison event without starving newer work', async () => { + await factories.createWorkflow({ + projectId, + enabled: true, + triggerType: 'EVENT', + triggerConfig: {eventName: 'poison.event'}, + }); + const contact = await factories.createContact({projectId}); + const poisonEvent = await prisma.event.create({ + data: { + projectId, + contactId: contact.id, + name: 'poison.event', + dispatchAttempts: 7, + createdAt: new Date(Date.now() - 10 * 60 * 1000), + }, + }); + const workflowLookup = vi + .spyOn(servicePrisma.workflow, 'findUnique') + .mockRejectedValue(new Error('permanently invalid workflow')); + + await expect(EventService.dispatchStoredEvent(poisonEvent.id)).rejects.toThrow('failed 1 dispatch target'); + + expect( + await prisma.event.findUniqueOrThrow({ + where: {id: poisonEvent.id}, + select: {processedAt: true, dispatchAttempts: true, dispatchFailedAt: true, nextDispatchAt: true}, + }), + ).toMatchObject({ + processedAt: null, + dispatchAttempts: 8, + dispatchFailedAt: expect.any(Date), + nextDispatchAt: null, + }); + + workflowLookup.mockRestore(); + const healthyEvent = await prisma.event.create({ + data: { + projectId, + contactId: contact.id, + name: 'healthy.event', + createdAt: new Date(Date.now() - 6 * 60 * 1000), + }, + }); + const updateProgress = vi.fn().mockResolvedValue(undefined); + + await processCleanup({updateProgress} as unknown as Job); + + expect((await prisma.event.findUniqueOrThrow({where: {id: healthyEvent.id}})).processedAt).not.toBeNull(); + expect((await prisma.event.findUniqueOrThrow({where: {id: poisonEvent.id}})).processedAt).toBeNull(); + }); +}); diff --git a/apps/api/src/controllers/__tests__/Webhooks.sns-idempotency.test.ts b/apps/api/src/controllers/__tests__/Webhooks.sns-idempotency.test.ts new file mode 100644 index 000000000..9f4049352 --- /dev/null +++ b/apps/api/src/controllers/__tests__/Webhooks.sns-idempotency.test.ts @@ -0,0 +1,458 @@ +import type {Request, Response} from 'express'; +import type {Job} from 'bullmq'; +import type {Prisma} from '@plunk/db'; +import type {IdempotencyKeyCleanupJobData} from '@plunk/types'; +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; + +import {factories, getPrismaClient} from '../../../../../test/helpers'; +import {prisma as controllerPrisma} from '../../database/prisma.js'; +import {processCleanup} from '../../jobs/idempotency-key-cleanup-processor.js'; +import {EventService} from '../../services/EventService.js'; +import {SecurityService} from '../../services/SecurityService.js'; +import {Webhooks} from '../Webhooks.js'; + +const SNS_TOPIC_ARN = 'arn:aws:sns:us-east-1:123456789012:plunk-ses-events'; + +function mockResponse() { + const captured = {status: 200, body: undefined as unknown}; + + let markSent: () => void; + const sent = new Promise(resolve => { + markSent = resolve; + }); + + const res = { + status(code: number) { + captured.status = code; + return this; + }, + json(body: unknown) { + captured.body = body; + markSent(); + return this; + }, + } as unknown as Response; + + return {res, captured, sent}; +} + +function notification( + snsMessageId: string, + sesMessageId: string, + eventType: 'Bounce' | 'Delivery' | 'Open' | 'Complaint' | 'Click' = 'Delivery', +): Request { + return { + body: { + Type: 'Notification', + MessageId: snsMessageId, + TopicArn: SNS_TOPIC_ARN, + Message: JSON.stringify({ + eventType, + mail: {messageId: sesMessageId}, + }), + }, + } as Request; +} + +function subscriptionConfirmation(): Request { + return { + body: { + Type: 'SubscriptionConfirmation', + MessageId: 'sns-subscription-confirmation', + TopicArn: SNS_TOPIC_ARN, + SubscribeURL: 'https://sns.eu-west-1.amazonaws.com/?Action=ConfirmSubscription', + }, + } as Request; +} + +function inboundNotification(snsMessageId: string, recipient: string): Request { + return { + body: { + Type: 'Notification', + MessageId: snsMessageId, + TopicArn: SNS_TOPIC_ARN, + Message: JSON.stringify({ + notificationType: 'Received', + mail: { + messageId: `ses-${snsMessageId}`, + source: 'Sender@external.example', + timestamp: new Date().toISOString(), + commonHeaders: {from: ['Sender '], subject: 'Inbound test'}, + }, + receipt: {recipients: [recipient]}, + }), + }, + } as Request; +} + +describe('SNS webhook delivery receipts', () => { + const prisma = getPrismaClient(); + const controller = new Webhooks(); + const next = vi.fn(); + + let projectId: string; + let contactId: string; + let sesMessageId: string; + + beforeEach(async () => { + const {project} = await factories.createUserWithProject(); + const contact = await factories.createContact({projectId: project.id}); + projectId = project.id; + contactId = contact.id; + sesMessageId = `ses-${project.id}`; + await factories.createEmail({projectId: project.id, contactId: contact.id, messageId: sesMessageId}); + + vi.spyOn(SecurityService, 'verifySnsSignature').mockResolvedValue(true); + vi.spyOn(EventService, 'dispatchStoredEvent').mockResolvedValue(undefined); + }); + + afterEach(async () => { + await prisma.snsWebhookReceipt.deleteMany(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + async function invoke(req: Request) { + const {res, captured, sent} = mockResponse(); + const handler = controller.receiveSNSWebhook as unknown as ( + req: Request, + res: Response, + next: (error?: unknown) => void, + ) => void; + handler(req, res, next); + await sent; + expect(next).not.toHaveBeenCalled(); + return captured; + } + + function deliver( + snsMessageId: string, + innerMessageId = sesMessageId, + eventType: 'Bounce' | 'Delivery' | 'Open' | 'Complaint' | 'Click' = 'Delivery', + ) { + return invoke(notification(snsMessageId, innerMessageId, eventType)); + } + + function failNextTransactionAfterEventInsert() { + const transaction = controllerPrisma.$transaction.bind(controllerPrisma); + vi.spyOn(controllerPrisma, '$transaction').mockImplementationOnce( + (async (callback: (tx: Prisma.TransactionClient) => Promise) => + transaction(async tx => { + const createEvent = tx.event.create.bind(tx.event); + vi.spyOn(tx.event, 'create').mockImplementationOnce(async args => { + await createEvent(args); + throw new Error('injected database failure after event insert'); + }); + return callback(tx); + })) as typeof controllerPrisma.$transaction, + ); + } + + it('acknowledges a completed replay without applying its effects twice', async () => { + const first = await deliver('sns-sequential-replay'); + const replay = await deliver('sns-sequential-replay'); + + expect(first.status).toBe(200); + expect(replay).toMatchObject({status: 200, body: {success: true, duplicate: true}}); + expect(SecurityService.verifySnsSignature).toHaveBeenCalledTimes(2); + expect(EventService.dispatchStoredEvent).toHaveBeenCalledTimes(1); + const email = await prisma.email.findUniqueOrThrow({where: {messageId: sesMessageId}}); + expect(await prisma.event.count({where: {emailId: email.id}})).toBe(1); + expect(await prisma.email.findUniqueOrThrow({where: {messageId: sesMessageId}})).toMatchObject({ + status: 'DELIVERED', + }); + + const receipt = await prisma.snsWebhookReceipt.findUniqueOrThrow({ + where: {messageId: 'sns-sequential-replay'}, + }); + expect(receipt.status).toBe('COMPLETED'); + expect(receipt.completedAt).not.toBeNull(); + expect(receipt.expiresAt.getTime()).toBeGreaterThan(Date.now() + 6 * 24 * 60 * 60 * 1000); + }); + + it('rejects an untrusted topic before parsing its nested message or claiming a receipt', async () => { + vi.mocked(SecurityService.verifySnsSignature).mockRestore(); + + const response = await invoke({ + body: { + Type: 'Notification', + MessageId: 'sns-untrusted-topic', + TopicArn: 'arn:aws:sns:us-east-1:999999999999:plunk-ses-events', + Message: '{not-valid-json', + }, + } as Request); + + expect(response).toMatchObject({status: 403, body: {success: false}}); + expect(EventService.dispatchStoredEvent).not.toHaveBeenCalled(); + expect(await prisma.snsWebhookReceipt.count()).toBe(0); + }); + + it('rejects an untrusted subscription before following its confirmation URL', async () => { + vi.mocked(SecurityService.verifySnsSignature).mockRestore(); + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + + const response = await invoke({ + body: { + Type: 'SubscriptionConfirmation', + MessageId: 'sns-untrusted-subscription', + TopicArn: 'arn:aws:sns:us-east-1:999999999999:plunk-ses-events', + SubscribeURL: 'https://sns.us-east-1.amazonaws.com/?Action=ConfirmSubscription', + }, + } as Request); + + expect(response).toMatchObject({status: 403, body: {success: false}}); + expect(fetchMock).not.toHaveBeenCalled(); + expect(await prisma.snsWebhookReceipt.count()).toBe(0); + }); + + it('rolls back partial email and event effects when the database fails, then retries once', async () => { + failNextTransactionAfterEventInsert(); + + const failed = await deliver('sns-db-failure', sesMessageId, 'Open'); + + expect(failed.status).toBe(500); + expect(EventService.dispatchStoredEvent).not.toHaveBeenCalled(); + expect(await prisma.email.findUniqueOrThrow({where: {messageId: sesMessageId}})).toMatchObject({ + status: 'PENDING', + opens: 0, + openedAt: null, + }); + expect(await prisma.event.count({where: {projectId, name: 'email.open'}})).toBe(0); + expect( + await prisma.snsWebhookReceipt.findUniqueOrThrow({where: {messageId: 'sns-db-failure'}}), + ).toMatchObject({status: 'FAILED', completedAt: null}); + + const retry = await deliver('sns-db-failure', sesMessageId, 'Open'); + + expect(retry.status).toBe(200); + expect(EventService.dispatchStoredEvent).toHaveBeenCalledTimes(1); + expect(await prisma.email.findUniqueOrThrow({where: {messageId: sesMessageId}})).toMatchObject({ + status: 'OPENED', + opens: 1, + }); + expect(await prisma.event.count({where: {projectId, name: 'email.open'}})).toBe(1); + expect( + await prisma.snsWebhookReceipt.findUniqueOrThrow({where: {messageId: 'sns-db-failure'}}), + ).toMatchObject({status: 'COMPLETED'}); + }); + + it('acknowledges after commit when workflow dispatch fails and leaves the event recoverable', async () => { + vi.mocked(EventService.dispatchStoredEvent).mockRejectedValueOnce(new Error('injected workflow failure')); + + const first = await deliver('sns-dispatch-failure'); + const replay = await deliver('sns-dispatch-failure'); + + expect(first.status).toBe(200); + expect(replay).toMatchObject({status: 200, body: {success: true, duplicate: true}}); + const event = await prisma.event.findFirstOrThrow({where: {projectId, name: 'email.delivery'}}); + expect(event.processedAt).toBeNull(); + expect(await prisma.event.count({where: {projectId, name: 'email.delivery'}})).toBe(1); + expect( + await prisma.snsWebhookReceipt.findUniqueOrThrow({where: {messageId: 'sns-dispatch-failure'}}), + ).toMatchObject({status: 'COMPLETED'}); + + vi.mocked(EventService.dispatchStoredEvent).mockRestore(); + await EventService.dispatchStoredEvent(event.id); + expect((await prisma.event.findUniqueOrThrow({where: {id: event.id}})).processedAt).not.toBeNull(); + }); + + it('rolls back inbound contact, email, and event effects before retrying once', async () => { + await factories.createDomain({projectId, domain: 'inbound.example', verified: true}); + failNextTransactionAfterEventInsert(); + + const failed = await invoke(inboundNotification('sns-inbound-db-failure', 'reply@inbound.example')); + + expect(failed.status).toBe(500); + expect( + await prisma.contact.findUnique({ + where: {projectId_email: {projectId, email: 'sender@external.example'}}, + }), + ).toBeNull(); + expect(await prisma.email.count({where: {projectId, sourceType: 'INBOUND'}})).toBe(0); + expect(await prisma.event.count({where: {projectId, name: 'email.received'}})).toBe(0); + expect( + await prisma.snsWebhookReceipt.findUniqueOrThrow({where: {messageId: 'sns-inbound-db-failure'}}), + ).toMatchObject({status: 'FAILED'}); + + const retry = await invoke(inboundNotification('sns-inbound-db-failure', 'reply@inbound.example')); + + expect(retry.status).toBe(200); + expect( + await prisma.contact.findUnique({ + where: {projectId_email: {projectId, email: 'sender@external.example'}}, + }), + ).not.toBeNull(); + expect(await prisma.email.count({where: {projectId, sourceType: 'INBOUND'}})).toBe(1); + expect(await prisma.event.count({where: {projectId, name: 'email.received'}})).toBe(1); + }); + + it('retries an event that arrives before its SES messageId is persisted', async () => { + const early = await deliver('sns-send-persist-race', 'ses-not-persisted-yet'); + + expect(early.status).toBe(500); + expect( + await prisma.snsWebhookReceipt.findUniqueOrThrow({where: {messageId: 'sns-send-persist-race'}}), + ).toMatchObject({status: 'FAILED'}); + + await factories.createEmail({projectId, contactId, messageId: 'ses-not-persisted-yet'}); + const retry = await deliver('sns-send-persist-race', 'ses-not-persisted-yet'); + + expect(retry.status).toBe(200); + expect(EventService.dispatchStoredEvent).toHaveBeenCalledTimes(1); + expect( + await prisma.snsWebhookReceipt.findUniqueOrThrow({where: {messageId: 'sns-send-persist-race'}}), + ).toMatchObject({status: 'COMPLETED'}); + }); + + it('reclaims an abandoned processing receipt instead of losing the delivery', async () => { + await prisma.snsWebhookReceipt.create({ + data: { + messageId: 'sns-abandoned-claim', + processingToken: 'abandoned-worker', + processingStartedAt: new Date(Date.now() - 10 * 60 * 1000), + expiresAt: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000), + }, + }); + + const retry = await deliver('sns-abandoned-claim'); + + expect(retry.status).toBe(200); + expect(EventService.dispatchStoredEvent).toHaveBeenCalledTimes(1); + expect( + await prisma.snsWebhookReceipt.findUniqueOrThrow({where: {messageId: 'sns-abandoned-claim'}}), + ).toMatchObject({status: 'COMPLETED'}); + }); + + it('does not reclaim a receipt after its owner renews the observed lease', async () => { + const originalToken = 'active-worker'; + await prisma.snsWebhookReceipt.create({ + data: { + messageId: 'sns-renewed-claim', + processingToken: originalToken, + processingStartedAt: new Date(Date.now() - 10 * 60 * 1000), + expiresAt: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000), + }, + }); + + const findReceipt = controllerPrisma.snsWebhookReceipt.findUnique.bind(controllerPrisma.snsWebhookReceipt); + vi.spyOn(controllerPrisma.snsWebhookReceipt, 'findUnique').mockImplementationOnce(async args => { + const observed = await findReceipt(args); + await prisma.snsWebhookReceipt.update({ + where: {messageId: 'sns-renewed-claim'}, + data: {processingStartedAt: new Date()}, + }); + return observed; + }); + + const response = await deliver('sns-renewed-claim'); + + expect(response.status).toBe(503); + expect(EventService.dispatchStoredEvent).not.toHaveBeenCalled(); + expect( + await prisma.snsWebhookReceipt.findUniqueOrThrow({where: {messageId: 'sns-renewed-claim'}}), + ).toMatchObject({status: 'PROCESSING', processingToken: originalToken}); + }); + + it('returns a retryable response to a concurrent duplicate', async () => { + let releaseUpdate!: () => void; + const updateGate = new Promise(resolve => { + releaseUpdate = resolve; + }); + let markUpdateStarted!: () => void; + const updateStarted = new Promise(resolve => { + markUpdateStarted = resolve; + }); + + const transaction = controllerPrisma.$transaction.bind(controllerPrisma); + vi.spyOn(controllerPrisma, '$transaction').mockImplementationOnce( + (async (callback: (tx: Prisma.TransactionClient) => Promise) => + transaction(async tx => { + const updateEmail = tx.email.update.bind(tx.email); + vi.spyOn(tx.email, 'update').mockImplementationOnce(async args => { + markUpdateStarted(); + await updateGate; + return updateEmail(args); + }); + return callback(tx); + })) as typeof controllerPrisma.$transaction, + ); + + const firstDelivery = deliver('sns-concurrent-replay'); + await updateStarted; + + const concurrentReplay = await deliver('sns-concurrent-replay'); + expect(concurrentReplay.status).toBe(503); + expect(EventService.dispatchStoredEvent).not.toHaveBeenCalled(); + + releaseUpdate(); + expect((await firstDelivery).status).toBe(200); + expect(EventService.dispatchStoredEvent).toHaveBeenCalledTimes(1); + }); + + it('keeps successful subscription confirmation behavior', async () => { + const fetchMock = vi.fn().mockResolvedValue({ok: true}); + vi.stubGlobal('fetch', fetchMock); + + const response = await invoke(subscriptionConfirmation()); + + expect(response).toMatchObject({status: 200, body: {success: true, message: 'Subscription confirmed'}}); + expect(fetchMock).toHaveBeenCalledWith( + 'https://sns.eu-west-1.amazonaws.com/?Action=ConfirmSubscription', + ); + expect(await prisma.snsWebhookReceipt.count()).toBe(0); + }); + + it('returns 5xx when subscription confirmation fails so SNS can retry', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ok: false, statusText: 'upstream failed'})); + + const response = await invoke(subscriptionConfirmation()); + + expect(response).toMatchObject({status: 502, body: {success: false}}); + expect(await prisma.snsWebhookReceipt.count()).toBe(0); + }); + + it('removes expired delivery receipts while retaining the replay window', async () => { + const now = Date.now(); + const pendingEvent = await prisma.event.create({ + data: { + projectId, + contactId, + name: 'outbox.recovery', + createdAt: new Date(now - 10 * 60 * 1000), + }, + }); + await prisma.snsWebhookReceipt.createMany({ + data: [ + { + messageId: 'sns-expired', + processingToken: 'expired', + status: 'COMPLETED', + completedAt: new Date(now - 8 * 24 * 60 * 60 * 1000), + expiresAt: new Date(now - 1_000), + }, + { + messageId: 'sns-retained', + processingToken: 'retained', + status: 'COMPLETED', + completedAt: new Date(now), + expiresAt: new Date(now + 7 * 24 * 60 * 60 * 1000), + }, + ], + }); + + vi.mocked(EventService.dispatchStoredEvent).mockRestore(); + const updateProgress = vi.fn().mockResolvedValue(undefined); + const result = await processCleanup({updateProgress} as unknown as Job); + + expect(result.deleted).toBe(1); + expect(updateProgress).toHaveBeenCalledWith(100); + expect((await prisma.event.findUniqueOrThrow({where: {id: pendingEvent.id}})).processedAt).not.toBeNull(); + await expect( + prisma.snsWebhookReceipt.findUnique({where: {messageId: 'sns-expired'}}), + ).resolves.toBeNull(); + await expect( + prisma.snsWebhookReceipt.findUnique({where: {messageId: 'sns-retained'}}), + ).resolves.not.toBeNull(); + }); +}); diff --git a/apps/api/src/controllers/__tests__/Webhooks.sns.test.ts b/apps/api/src/controllers/__tests__/Webhooks.sns.test.ts index 658b8ac0a..8f393ffe7 100644 --- a/apps/api/src/controllers/__tests__/Webhooks.sns.test.ts +++ b/apps/api/src/controllers/__tests__/Webhooks.sns.test.ts @@ -1,4 +1,5 @@ import type {Request, Response} from 'express'; +import {randomUUID} from 'node:crypto'; import {beforeEach, describe, expect, it, vi} from 'vitest'; import {EmailStatus} from '@plunk/db'; @@ -71,10 +72,10 @@ describe('Webhooks - SES event notifications', () => { } /** Deliver one SES event notification, shaped the way SNS posts it. */ - async function post(event: Record) { + async function post(event: Record, snsMessageId = randomUUID()) { const {res, captured, sent} = mockResponse(); const req = { - body: {Type: 'Notification', Message: JSON.stringify(event)}, + body: {Type: 'Notification', MessageId: snsMessageId, Message: JSON.stringify(event)}, get: () => undefined, headers: {}, } as unknown as Request; @@ -242,8 +243,10 @@ describe('Webhooks - SES event notifications', () => { await sentEmail('ses-campaign-2', campaign.id); await prisma.campaign.update({where: {id: campaign.id}, data: {sentCount: 1}}); - await post(notification('Delivery', 'ses-campaign-2')); - await post(notification('Delivery', 'ses-campaign-2')); + const replayedNotification = notification('Delivery', 'ses-campaign-2'); + const replayedSnsMessageId = randomUUID(); + await post(replayedNotification, replayedSnsMessageId); + await post(replayedNotification, replayedSnsMessageId); await CampaignService.sweepDirtyStats(100); const stats = await CampaignService.getStats(projectId, campaign.id); @@ -272,10 +275,10 @@ describe('Webhooks - SES event notifications', () => { expect(updated?.deliveredAt).toBeNull(); }); - it('404s an event for a messageId it does not know', async () => { + it('retries an event for a messageId it does not know', async () => { const captured = await post(notification('Delivery', 'ses-never-sent')); - expect(captured.status).toBe(404); + expect(captured.status).toBe(500); }); }); }); diff --git a/apps/api/src/jobs/idempotency-key-cleanup-processor.ts b/apps/api/src/jobs/idempotency-key-cleanup-processor.ts index 8750f27c9..9c8b9b5e9 100644 --- a/apps/api/src/jobs/idempotency-key-cleanup-processor.ts +++ b/apps/api/src/jobs/idempotency-key-cleanup-processor.ts @@ -6,20 +6,26 @@ import signale from 'signale'; import {REDIS_URL} from '../app/constants.js'; import {prisma} from '../database/prisma.js'; +import {EventService} from '../services/EventService.js'; /** - * Idempotency Key Cleanup Worker - * Deletes claims past their expiresAt, which both bounds table growth and is what - * makes an expired key reusable. Runs hourly, since the TTL is measured in hours. + * Durable-state maintenance worker. + * + * Deletes expired API/SNS claims and reconciles aged event-outbox rows. The + * five-minute grace window keeps the minutely sweep away from normal + * synchronous dispatch while bounding the usual recovery delay below six + * minutes. */ const BATCH_SIZE = 10000; // Delete in batches to avoid long-held locks +const EVENT_DISPATCH_BATCH_SIZE = 100; +const EVENT_DISPATCH_GRACE_MS = 5 * 60 * 1000; /** * Process idempotency key cleanup job */ -async function processCleanup(job: Job): Promise<{deleted: number}> { - signale.info('[IDEMPOTENCY-CLEANUP] Starting cleanup of expired idempotency keys...'); +export async function processCleanup(job: Job): Promise<{deleted: number}> { + signale.info('[IDEMPOTENCY-CLEANUP] Starting durable-state maintenance...'); let totalDeleted = 0; @@ -46,7 +52,60 @@ async function processCleanup(job: Job): Promise<{ await new Promise(resolve => setTimeout(resolve, 100)); } - signale.success(`[IDEMPOTENCY-CLEANUP] Cleanup complete. Deleted ${totalDeleted} expired keys`); + for (;;) { + const deleted = await prisma.$executeRaw` + DELETE FROM "sns_webhook_receipts" + WHERE "id" IN ( + SELECT "id" FROM "sns_webhook_receipts" + WHERE "expiresAt" < NOW() + LIMIT ${BATCH_SIZE} + ) + `; + + totalDeleted += deleted; + + if (deleted < BATCH_SIZE) { + break; + } + + signale.info(`[IDEMPOTENCY-CLEANUP] Deleted ${totalDeleted} claims so far, continuing...`); + await new Promise(resolve => setTimeout(resolve, 100)); + } + + const reconciliationStartedAt = new Date(); + const pendingEvents = await prisma.event.findMany({ + where: { + processedAt: null, + dispatchFailedAt: null, + createdAt: {lt: new Date(Date.now() - EVENT_DISPATCH_GRACE_MS)}, + AND: [ + {OR: [{nextDispatchAt: null}, {nextDispatchAt: {lte: reconciliationStartedAt}}]}, + {OR: [{dispatchLeaseExpiresAt: null}, {dispatchLeaseExpiresAt: {lte: reconciliationStartedAt}}]}, + ], + }, + select: {id: true}, + orderBy: {createdAt: 'asc'}, + take: EVENT_DISPATCH_BATCH_SIZE, + }); + let dispatchedEvents = 0; + + for (const event of pendingEvents) { + try { + await EventService.dispatchStoredEvent(event.id); + dispatchedEvents += 1; + } catch (error) { + // dispatchStoredEvent recorded backoff or terminal failure. Keep the + // event durable without letting it monopolize every sweep. + signale.error(`[EVENT-OUTBOX] Failed to dispatch event ${event.id}:`, error); + } + } + + signale.success(`[IDEMPOTENCY-CLEANUP] Cleanup complete. Deleted ${totalDeleted} expired records`); + if (pendingEvents.length > 0) { + signale.info( + `[EVENT-OUTBOX] Dispatched ${dispatchedEvents}/${pendingEvents.length} pending events; failures retry next sweep`, + ); + } await job.updateProgress(100); diff --git a/apps/api/src/services/EventService.ts b/apps/api/src/services/EventService.ts index 0a06b15d6..9a55d43aa 100644 --- a/apps/api/src/services/EventService.ts +++ b/apps/api/src/services/EventService.ts @@ -1,3 +1,5 @@ +import {randomUUID} from 'node:crypto'; + import type {Event} from '@plunk/db'; import {Prisma} from '@plunk/db'; import type {FilterCondition, FilterGroup} from '@plunk/types'; @@ -10,6 +12,17 @@ import {Keys} from './keys.js'; import {WorkflowExecutionService} from './WorkflowExecutionService.js'; +const EVENT_DISPATCH_MAX_ATTEMPTS = 8; +const EVENT_DISPATCH_BASE_DELAY_MS = 60 * 1000; +const EVENT_DISPATCH_MAX_DELAY_MS = 60 * 60 * 1000; +const EVENT_DISPATCH_LEASE_MS = 15 * 60 * 1000; +const EVENT_DISPATCH_HEARTBEAT_MS = EVENT_DISPATCH_LEASE_MS / 3; + +type DispatchLeaseGuard = { + assertOwnership: () => Promise; + stop: () => void; +}; + /** * Event Service * Handles event tracking and workflow triggering @@ -37,15 +50,199 @@ export class EventService { }, }); - // Trigger workflows that are listening for this event - await this.triggerWorkflows(projectId, eventName, contactId, data); - - // Resume workflows waiting for this event - await WorkflowExecutionService.handleEvent(projectId, eventName, contactId, data); + try { + await this.dispatchStoredEvent(event.id); + } catch (error) { + // The event is already committed. Keep processedAt null and acknowledge + // ingestion; the bounded reconciliation sweep owns workflow delivery. + signale.error(`[EVENT-OUTBOX] Event ${event.id} is stored but dispatch failed:`, error); + } return event; } + /** + * Dispatch a durably stored event to workflow triggers and waits. A failed + * dispatch leaves processedAt null so the reconciliation worker can retry it. + * The fenced lease keeps a sweep from overlapping live request dispatch. + */ + public static async dispatchStoredEvent(eventId: string): Promise { + const leaseId = randomUUID(); + const now = new Date(); + const claimed = await prisma.event.updateMany({ + where: { + id: eventId, + processedAt: null, + dispatchFailedAt: null, + OR: [{dispatchLeaseExpiresAt: null}, {dispatchLeaseExpiresAt: {lte: now}}], + }, + data: { + dispatchLeaseId: leaseId, + dispatchLeaseExpiresAt: new Date(now.getTime() + EVENT_DISPATCH_LEASE_MS), + }, + }); + if (claimed.count === 0) return; + + const lease = this.startDispatchLeaseHeartbeat(eventId, leaseId); + try { + const event = await prisma.event.findUnique({where: {id: eventId}}); + if (!event) return; + + const data = + event.data && typeof event.data === 'object' && !Array.isArray(event.data) + ? (event.data as Record) + : undefined; + + let errors: unknown[] = []; + try { + errors = await this.triggerWorkflows( + event.id, + event.projectId, + event.name, + event.contactId ?? undefined, + data, + lease.assertOwnership, + ); + } catch (error) { + errors.push(error); + } + + try { + await lease.assertOwnership(); + await WorkflowExecutionService.handleEvent( + event.projectId, + event.name, + event.contactId ?? undefined, + data, + event.id, + lease.assertOwnership, + ); + } catch (error) { + errors.push(error); + } + + if (errors.length > 0) { + throw new AggregateError(errors, `Event ${event.id} failed ${errors.length} dispatch target(s)`); + } + + await lease.assertOwnership(); + const completed = await prisma.event.updateMany({ + where: {id: event.id, dispatchLeaseId: leaseId, processedAt: null, dispatchFailedAt: null}, + data: { + processedAt: new Date(), + nextDispatchAt: null, + dispatchError: null, + dispatchLeaseId: null, + dispatchLeaseExpiresAt: null, + }, + }); + + if (completed.count !== 1) { + throw new Error(`Event ${event.id} dispatch lease was lost before acknowledgement`); + } + } catch (error) { + await this.recordDispatchFailure(eventId, leaseId, error); + throw error; + } finally { + lease.stop(); + } + } + + /** + * Renew a fenced lease while workflow continuations are running. A failed + * heartbeat permanently invalidates this dispatcher so it cannot keep + * delivering after another worker may have reclaimed the event. + */ + private static startDispatchLeaseHeartbeat( + eventId: string, + leaseId: string, + intervalMs = EVENT_DISPATCH_HEARTBEAT_MS, + ): DispatchLeaseGuard { + let heartbeatError: unknown; + let renewal: Promise | undefined; + + const renew = async (): Promise => { + if (heartbeatError) throw heartbeatError; + if (renewal) return renewal; + + renewal = (async () => { + const now = new Date(); + const renewed = await prisma.event.updateMany({ + where: {id: eventId, dispatchLeaseId: leaseId, processedAt: null, dispatchFailedAt: null}, + data: {dispatchLeaseExpiresAt: new Date(now.getTime() + EVENT_DISPATCH_LEASE_MS)}, + }); + if (renewed.count !== 1) { + throw new Error(`Event ${eventId} dispatch lease is no longer owned by ${leaseId}`); + } + })() + .catch(error => { + heartbeatError = error; + throw error; + }) + .finally(() => { + renewal = undefined; + }); + + return renewal; + }; + + const timer = setInterval(() => { + void renew().catch(error => { + signale.error(`[EVENT-OUTBOX] Event ${eventId} dispatch lease heartbeat failed:`, error); + }); + }, intervalMs); + timer.unref(); + + return { + assertOwnership: renew, + stop: () => clearInterval(timer), + }; + } + + /** + * Back off repeated failures and dead-letter a poison event after a bounded + * number of attempts. Terminal rows remain inspectable but leave the sweep. + */ + private static async recordDispatchFailure(eventId: string, leaseId: string, error: unknown): Promise { + const event = await prisma.event.findUnique({ + where: {id: eventId}, + select: {dispatchAttempts: true, dispatchLeaseId: true, processedAt: true, dispatchFailedAt: true}, + }); + if (!event || event.dispatchLeaseId !== leaseId || event.processedAt || event.dispatchFailedAt) return; + + const attempts = event.dispatchAttempts + 1; + const terminal = attempts >= EVENT_DISPATCH_MAX_ATTEMPTS; + const delayMs = Math.min( + EVENT_DISPATCH_BASE_DELAY_MS * 2 ** Math.max(0, attempts - 1), + EVENT_DISPATCH_MAX_DELAY_MS, + ); + const message = ( + error instanceof AggregateError + ? `${error.message}: ${error.errors + .map(cause => (cause instanceof Error ? cause.message : String(cause))) + .join('; ')}` + : error instanceof Error + ? error.message + : String(error) + ).slice(0, 2000); + + const updated = await prisma.event.updateMany({ + where: {id: eventId, dispatchLeaseId: leaseId, processedAt: null, dispatchFailedAt: null}, + data: { + dispatchAttempts: attempts, + dispatchError: message, + nextDispatchAt: terminal ? null : new Date(Date.now() + delayMs), + dispatchFailedAt: terminal ? new Date() : null, + dispatchLeaseId: null, + dispatchLeaseExpiresAt: null, + }, + }); + + if (terminal && updated.count > 0) { + signale.error(`[EVENT-OUTBOX] Event ${eventId} exhausted ${attempts} dispatch attempts and was dead-lettered`); + } + } + /** * Invalidate the workflow cache for a project * Should be called when workflows are enabled/disabled or updated @@ -349,11 +546,14 @@ export class EventService { * Uses Redis caching for enabled workflows to improve performance */ private static async triggerWorkflows( + eventId: string, projectId: string, eventName: string, contactId?: string, data?: Record, - ): Promise { + assertDispatchOwnership?: () => Promise, + ): Promise { + const errors: unknown[] = []; // Try to get workflows from cache const cacheKey = Keys.Workflow.enabled(projectId); let workflows; @@ -395,9 +595,16 @@ export class EventService { // Check if this workflow is triggered by this event if (triggerConfig?.eventName === eventName) { + await assertDispatchOwnership?.(); + // If event is for a specific contact, start workflow for that contact if (contactId) { - await this.startWorkflowForContact(workflow.id, contactId, data); + try { + await this.startWorkflowForContact(workflow.id, contactId, eventId, data); + } catch (error) { + signale.error(`[EVENT] Failed to enroll workflow ${workflow.id} from event ${eventId}:`, error); + errors.push(error); + } } else { // If event is not contact-specific, you might want different logic // For example, trigger for all contacts, or skip @@ -405,6 +612,8 @@ export class EventService { } } } + + return errors; } /** @@ -413,93 +622,121 @@ export class EventService { private static async startWorkflowForContact( workflowId: string, contactId: string, + sourceEventId: string, context?: Record, ): Promise { - try { - // Get workflow with steps and configuration - const workflow = await prisma.workflow.findUnique({ - where: {id: workflowId}, - include: { - steps: { - where: {type: 'TRIGGER'}, - }, + // Get workflow with steps and configuration + const workflow = await prisma.workflow.findUnique({ + where: {id: workflowId}, + include: { + steps: { + where: {type: 'TRIGGER'}, }, - }); + }, + }); - if (!workflow || workflow.steps.length === 0) { - signale.error(`[EVENT] Workflow ${workflowId} has no trigger step`); - return; - } + if (!workflow || workflow.steps.length === 0) { + signale.error(`[EVENT] Workflow ${workflowId} has no trigger step`); + return; + } - // Never run a workflow against a contact from another project. - // Queried directly instead of via ContactService, which imports this service. - const contact = await prisma.contact.findFirst({ - where: {id: contactId, projectId: workflow.projectId}, - select: {id: true}, - }); + // Never run a workflow against a contact from another project. + // Queried directly instead of via ContactService, which imports this service. + const contact = await prisma.contact.findFirst({ + where: {id: contactId, projectId: workflow.projectId}, + select: {id: true}, + }); - if (!contact) { - signale.warn( - `[EVENT] Refusing to start workflow ${workflowId} for contact ${contactId}: contact does not belong to project ${workflow.projectId}`, - ); - return; - } + if (!contact) { + signale.warn( + `[EVENT] Refusing to start workflow ${workflowId} for contact ${contactId}: contact does not belong to project ${workflow.projectId}`, + ); + return; + } - // Check re-entry rules - if (!workflow.allowReentry) { - // If re-entry is not allowed, check if contact has ANY execution (regardless of status) - const existingExecution = await prisma.workflowExecution.findFirst({ - where: { - workflowId, - contactId, - }, - }); + const triggerStep = workflow.steps[0]; - if (existingExecution) { - return; - } - } else { - // If re-entry is allowed, only check if there's a currently RUNNING execution - const runningExecution = await prisma.workflowExecution.findFirst({ - where: { - workflowId, - contactId, - status: 'RUNNING', - }, - }); + if (!triggerStep) { + signale.error(`[EVENT] Workflow ${workflowId} trigger step not found`); + return; + } - if (runningExecution) { - return; - } + // Once a step has begun, enrollment is durably delivered. Do not replay a + // failed step: its external effect may have succeeded before the engine + // recorded the failure. A RUNNING execution with no step row is safe to + // resume because no workflow step has started yet. + const eventExecution = await prisma.workflowExecution.findFirst({ + where: {workflowId, sourceEventId}, + select: { + id: true, + status: true, + stepExecutions: {select: {id: true}, take: 1}, + }, + }); + + if (eventExecution) { + if (eventExecution.status === 'RUNNING' && eventExecution.stepExecutions.length === 0) { + await WorkflowExecutionService.processStepExecution(eventExecution.id, triggerStep.id); } + return; + } - const triggerStep = workflow.steps[0]; + // Check re-entry rules + if (!workflow.allowReentry) { + // If re-entry is not allowed, check if contact has ANY execution (regardless of status) + const existingExecution = await prisma.workflowExecution.findFirst({ + where: { + workflowId, + contactId, + }, + }); - if (!triggerStep) { - signale.error(`[EVENT] Workflow ${workflowId} trigger step not found`); + if (existingExecution) { return; } + } else { + // If re-entry is allowed, only check if there's a currently RUNNING execution + const runningExecution = await prisma.workflowExecution.findFirst({ + where: { + workflowId, + contactId, + status: 'RUNNING', + }, + }); + + if (runningExecution) { + return; + } + } - // Create workflow execution - const execution = await prisma.workflowExecution.create({ + let execution; + try { + execution = await prisma.workflowExecution.create({ data: { workflowId, contactId, status: 'RUNNING', currentStepId: triggerStep.id, + sourceEventId, context: context ? toPrismaJson(context) : undefined, }, }); - - signale.info( - `[EVENT] Started workflow ${workflowId} execution ${execution.id} for contact ${contactId}${workflow.allowReentry ? ' (re-entry allowed)' : ''}`, - ); - - // Start executing the workflow - await WorkflowExecutionService.processStepExecution(execution.id, triggerStep.id); } catch (error) { - signale.error(`[EVENT] Error starting workflow ${workflowId}:`, error); + // Concurrent dispatchers may both observe no enrollment. The database + // chooses one winner; the other has nothing left to deliver. + if (error instanceof Error && 'code' in error && error.code === 'P2002') { + return; + } + throw error; } + + signale.info( + `[EVENT] Started workflow ${workflowId} execution ${execution.id} for contact ${contactId}${workflow.allowReentry ? ' (re-entry allowed)' : ''}`, + ); + + // A dispatch failure must propagate so the event stays unprocessed. The + // request still acknowledges the committed event; maintenance retries it. + await WorkflowExecutionService.processStepExecution(execution.id, triggerStep.id); } /** diff --git a/apps/api/src/services/SecurityService.ts b/apps/api/src/services/SecurityService.ts index 4f3bad664..1512f36c6 100644 --- a/apps/api/src/services/SecurityService.ts +++ b/apps/api/src/services/SecurityService.ts @@ -21,6 +21,7 @@ import { PHISHING_CUMULATIVE_WINDOW_MS, PHISHING_DETECTION_ENABLED, PHISHING_DETECTION_SAMPLE_RATE, + SNS_TOPIC_ARNS, } from '../app/constants.js'; /** @@ -171,11 +172,17 @@ export class SecurityService { private static readonly CACHE_TTL = 300; // 5 minutes /** - * Verify an AWS SNS message signature. Returns false if the cert URL is - * untrusted, or the signature doesn't match. + * Authorize the signed topic and verify its AWS SNS signature. Returns false + * before certificate I/O when the topic is not configured for this deployment. */ public static async verifySnsSignature(body: Record): Promise { try { + const topicArn = body['TopicArn']; + if (typeof topicArn !== 'string' || !SNS_TOPIC_ARNS.has(topicArn)) { + signale.warn('[SNS] Missing or untrusted TopicArn'); + return false; + } + const certUrl = body['SigningCertURL']; const signature = body['Signature']; diff --git a/apps/api/src/services/WorkflowExecutionService.ts b/apps/api/src/services/WorkflowExecutionService.ts index b62cf0f9b..06cf59aa0 100644 --- a/apps/api/src/services/WorkflowExecutionService.ts +++ b/apps/api/src/services/WorkflowExecutionService.ts @@ -7,7 +7,7 @@ import type { WorkflowStep, WorkflowStepExecution, } from '@plunk/db'; -import {StepExecutionStatus, WorkflowExecutionStatus} from '@plunk/db'; +import {StepExecutionStatus, WorkflowExecutionStatus, WorkflowWaitOutcome} from '@plunk/db'; import {toPrismaJson} from '@plunk/types'; import {renderTemplate, WorkflowStepConfigSchemas} from '@plunk/shared'; import dns from 'node:dns/promises'; @@ -30,6 +30,7 @@ type WorkflowStepWithTransitions = WorkflowStep & { outgoingTransitions?: Array<{ id: string; condition: Prisma.JsonValue; + waitOutcome: WorkflowWaitOutcome | null; priority: number; toStep: WorkflowStep; }>; @@ -328,71 +329,28 @@ export class WorkflowExecutionService { return; } + const result = { + waitOutcome: WorkflowWaitOutcome.TIMEOUT, + timedOut: true, + eventName: + stepExecution.step.config && + typeof stepExecution.step.config === 'object' && + 'eventName' in stepExecution.step.config + ? stepExecution.step.config.eventName + : undefined, + }; + // Mark step as completed with timeout await prisma.workflowStepExecution.update({ where: {id: stepExecution.id}, data: { status: StepExecutionStatus.COMPLETED, completedAt: new Date(), - output: { - timedOut: true, - eventName: - stepExecution.step.config && - typeof stepExecution.step.config === 'object' && - 'eventName' in stepExecution.step.config - ? stepExecution.step.config.eventName - : undefined, - }, + output: toPrismaJson(result), }, }); - // Continue workflow - find transitions with timeout/fallback logic - const transitions = stepExecution.step.outgoingTransitions || []; - const fallbackTransition = transitions.find( - t => - (t.condition && - typeof t.condition === 'object' && - 'branch' in t.condition && - t.condition.branch === 'timeout') || - (t.condition && typeof t.condition === 'object' && 'fallback' in t.condition && t.condition.fallback === true), - ); - - if (fallbackTransition) { - // Follow timeout branch - await prisma.workflowExecution.update({ - where: {id: stepExecution.executionId}, - data: { - status: WorkflowExecutionStatus.RUNNING, - currentStepId: fallbackTransition.toStep.id, - }, - }); - - await this.processStepExecution(stepExecution.executionId, fallbackTransition.toStep.id); - } else if (transitions.length > 0) { - // No timeout branch, follow first transition - const firstTransition = transitions[0]; - if (firstTransition?.toStep) { - const nextStep = firstTransition.toStep; - await prisma.workflowExecution.update({ - where: {id: stepExecution.executionId}, - data: { - status: WorkflowExecutionStatus.RUNNING, - currentStepId: nextStep.id, - }, - }); - - await this.processStepExecution(stepExecution.executionId, nextStep.id); - } - } else { - // No transitions, complete workflow - await prisma.workflowExecution.update({ - where: {id: stepExecution.executionId}, - data: { - status: WorkflowExecutionStatus.COMPLETED, - completedAt: new Date(), - }, - }); - } + await this.processNextSteps(stepExecution.execution, stepExecution.step, result); } /** @@ -403,11 +361,21 @@ export class WorkflowExecutionService { eventName: string, contactId?: string, data?: Record, + sourceEventId?: string, + assertDispatchOwnership?: () => Promise, ): Promise { - // Find workflows waiting for this event + const errors: unknown[] = []; + + // A durable retry must see both unclaimed waits and waits this same event + // already claimed before its continuation failed. const waitingExecutions = await prisma.workflowStepExecution.findMany({ where: { - status: StepExecutionStatus.WAITING, + OR: sourceEventId + ? [ + {status: StepExecutionStatus.WAITING, resumeEventId: null}, + {status: StepExecutionStatus.RUNNING, resumeEventId: sourceEventId}, + ] + : [{status: StepExecutionStatus.WAITING}], execution: { workflow: {projectId}, ...(contactId ? {contactId} : {}), @@ -438,27 +406,115 @@ export class WorkflowExecutionService { const config = stepExecution.step.config; if (config && typeof config === 'object' && 'eventName' in config && config.eventName === eventName) { - // Event matches, resume execution - await prisma.workflowStepExecution.update({ - where: {id: stepExecution.id}, - data: { - status: StepExecutionStatus.COMPLETED, - completedAt: new Date(), - output: toPrismaJson({ + await assertDispatchOwnership?.(); + + try { + let result: StepResult; + + if (sourceEventId && stepExecution.resumeEventId === sourceEventId) { + if ( + !stepExecution.output || + typeof stepExecution.output !== 'object' || + Array.isArray(stepExecution.output) + ) { + throw new Error(`Wait execution ${stepExecution.id} has no durable resume payload`); + } + result = stepExecution.output as StepResult; + + // processNextSteps may have returned before the wait acknowledgement + // was persisted. Its downstream step row is the durable delivery + // claim; never replay that step just to repair this wait row. + if (await this.waitContinuationWasDelivered(stepExecution, result)) { + await prisma.workflowStepExecution.updateMany({ + where: { + id: stepExecution.id, + status: StepExecutionStatus.RUNNING, + resumeEventId: sourceEventId, + }, + data: { + status: StepExecutionStatus.COMPLETED, + completedAt: new Date(), + }, + }); + continue; + } + } else { + const currentContext = + stepExecution.execution.context && + typeof stepExecution.execution.context === 'object' && + !Array.isArray(stepExecution.execution.context) + ? (stepExecution.execution.context as Record) + : {}; + const resumedContext = {...currentContext, ...(data ?? {})}; + result = { + waitOutcome: WorkflowWaitOutcome.EVENT, + eventReceived: true, eventName, - eventData: data ? toPrismaJson(data) : undefined, + ...(data ? {eventData: data} : {}), receivedAt: new Date().toISOString(), - }), - }, - }); - - // Cancel any pending timeout job - await QueueService.cancelWorkflowTimeout(stepExecution.id); - - // Continue workflow - await this.processNextSteps(stepExecution.execution, stepExecution.step, {eventReceived: true}); + }; + + // RUNNING is the durable event-vs-timeout claim. The row stays + // discoverable by resumeEventId until its continuation succeeds. + const claimed = await prisma.$transaction(async tx => { + const claim = await tx.workflowStepExecution.updateMany({ + where: { + id: stepExecution.id, + status: StepExecutionStatus.WAITING, + ...(sourceEventId ? {resumeEventId: null} : {}), + }, + data: { + status: StepExecutionStatus.RUNNING, + ...(sourceEventId ? {resumeEventId: sourceEventId} : {}), + completedAt: null, + output: toPrismaJson(result), + }, + }); + if (claim.count === 0) return false; + + await tx.workflowExecution.update({ + where: {id: stepExecution.executionId}, + data: {context: toPrismaJson(resumedContext)}, + }); + return true; + }); + + // Another event or the timeout won the compare-and-set. + if (!claimed) continue; + } + + // Cancel any pending timeout job + await QueueService.cancelWorkflowTimeout(stepExecution.id); + + // Continue workflow + await this.processNextSteps(stepExecution.execution, stepExecution.step, result); + + // Only acknowledge the wait after its continuation returns. If either + // operation above fails, the source event retries this claimed row. + await prisma.workflowStepExecution.updateMany({ + where: { + id: stepExecution.id, + status: StepExecutionStatus.RUNNING, + ...(sourceEventId ? {resumeEventId: sourceEventId} : {}), + }, + data: { + status: StepExecutionStatus.COMPLETED, + completedAt: new Date(), + }, + }); + } catch (error) { + signale.error( + `[WORKFLOW] Failed to resume execution ${stepExecution.executionId} from event ${eventName}:`, + error, + ); + errors.push(error); + } } } + + if (errors.length > 0) { + throw new AggregateError(errors, `Event ${eventName} failed ${errors.length} waiting execution(s)`); + } } /** @@ -1192,7 +1248,7 @@ export class WorkflowExecutionService { * Process next steps based on transitions */ private static async processNextSteps( - execution: WorkflowExecutionWithRelations, + execution: Pick, currentStep: WorkflowStepWithTransitions, stepResult: StepResult, ): Promise { @@ -1211,10 +1267,15 @@ export class WorkflowExecutionService { return; } - // Find the appropriate transition based on conditions + // WAIT_FOR_EVENT owns a distinct routing dimension. It does not borrow the + // condition JSON used by CONDITION steps. let nextStep = null; - for (const transition of transitions) { + if (currentStep.type === 'WAIT_FOR_EVENT') { + nextStep = this.getWaitContinuationStep(currentStep, stepResult); + } + + for (const transition of currentStep.type === 'WAIT_FOR_EVENT' ? [] : transitions) { const condition = transition.condition; // If no condition, always follow @@ -1271,6 +1332,55 @@ export class WorkflowExecutionService { await this.processStepExecution(execution.id, nextStep.id); } + /** + * Resolve a wait's event/timeout route without mutating execution state. + */ + private static getWaitContinuationStep( + currentStep: WorkflowStepWithTransitions, + stepResult: StepResult, + ): WorkflowStep | null { + const transitions = currentStep.outgoingTransitions || []; + const outcomeTransition = transitions.find(t => t.waitOutcome === stepResult.waitOutcome); + + if (outcomeTransition) return outcomeTransition.toStep; + + // Compatibility for rows created before the wait-outcome migration and + // direct test fixtures: their single transition handled either result. + return transitions.every(t => t.waitOutcome === null) ? (transitions[0]?.toStep ?? null) : null; + } + + /** + * A downstream step row means this wait already handed control to the engine. + * If there is no route, the terminal workflow update is the delivery proof. + */ + private static async waitContinuationWasDelivered( + waitExecution: WorkflowStepExecution & { + execution: WorkflowExecution; + step: WorkflowStepWithTransitions; + }, + result: StepResult, + ): Promise { + const nextStep = this.getWaitContinuationStep(waitExecution.step, result); + + if (nextStep) { + const downstreamClaim = await prisma.workflowStepExecution.findFirst({ + where: { + executionId: waitExecution.executionId, + stepId: nextStep.id, + createdAt: {gte: waitExecution.updatedAt}, + }, + select: {id: true}, + }); + return downstreamClaim !== null; + } + + const execution = await prisma.workflowExecution.findUnique({ + where: {id: waitExecution.executionId}, + select: {currentStepId: true, status: true}, + }); + return execution?.currentStepId === null && execution.status === WorkflowExecutionStatus.COMPLETED; + } + /** * Helper: Render template with variables * Uses shared template rendering from @plunk/shared @@ -1372,7 +1482,7 @@ export class WorkflowExecutionService { private static evaluateTransitionCondition( _condition: Prisma.JsonValue, _stepResult: StepResult, - _execution: WorkflowExecutionWithRelations, + _execution: Pick, ): boolean { // Implement custom transition condition logic here // For now, return false as default diff --git a/apps/api/src/services/WorkflowService.ts b/apps/api/src/services/WorkflowService.ts index 47aafb48b..5258fe1c5 100644 --- a/apps/api/src/services/WorkflowService.ts +++ b/apps/api/src/services/WorkflowService.ts @@ -1,5 +1,5 @@ import type {Workflow, WorkflowExecution, WorkflowStep, WorkflowTransition} from '@plunk/db'; -import {Prisma, WorkflowExecutionStatus} from '@plunk/db'; +import {Prisma, WorkflowExecutionStatus, WorkflowWaitOutcome} from '@plunk/db'; import type {PaginatedResponse, WorkflowExecutionWithDetails, WorkflowWithDetails} from '@plunk/types'; import {toPrismaJson} from '@plunk/types'; import signale from 'signale'; @@ -14,6 +14,56 @@ import {NtfyService} from './NtfyService.js'; import {WorkflowExecutionService} from './WorkflowExecutionService.js'; export class WorkflowService { + private static async validateStepTemplate(projectId: string, templateId?: string | null): Promise { + if (templateId === undefined || templateId === null) { + return; + } + + const template = await prisma.template.findFirst({ + where: {id: templateId, projectId}, + select: {id: true}, + }); + + if (!template) { + throw new HttpException(404, 'Template not found'); + } + } + + private static waitHasTimeout(config: Prisma.JsonValue): boolean { + return ( + typeof config === 'object' && + config !== null && + !Array.isArray(config) && + typeof config.timeout === 'number' && + config.timeout > 0 + ); + } + + /** + * Build the default route(s) when a step is inserted into a linear path. + * WAIT_FOR_EVENT keeps event and timeout routing explicit even when both + * outcomes continue to the same next step. + */ + private static defaultOutgoingTransitions( + step: Pick, + toStepId: string, + ): Prisma.WorkflowTransitionCreateManyInput[] { + const base = {fromStepId: step.id, toStepId, priority: 0}; + + if (step.type === 'CONDITION') { + return [{...base, condition: toPrismaJson({branch: 'yes'})}]; + } + + if (step.type === 'WAIT_FOR_EVENT') { + return [ + {...base, waitOutcome: WorkflowWaitOutcome.EVENT}, + ...(this.waitHasTimeout(step.config) ? [{...base, waitOutcome: WorkflowWaitOutcome.TIMEOUT, priority: 1}] : []), + ]; + } + + return [base]; + } + /** * Get all workflows for a project with pagination */ @@ -469,6 +519,7 @@ export class WorkflowService { transition.condition === null ? Prisma.JsonNull : (transition.condition as Prisma.InputJsonValue), + waitOutcome: transition.waitOutcome, priority: transition.priority, }, }); @@ -504,6 +555,8 @@ export class WorkflowService { } } + await this.validateStepTemplate(projectId, data.templateId); + // Create the new step const newStep = await prisma.workflowStep.create({ data: { @@ -530,12 +583,8 @@ export class WorkflowService { const lastStep = stepsWithoutOutgoing[stepsWithoutOutgoing.length - 1]; if (lastStep) { - await prisma.workflowTransition.create({ - data: { - fromStepId: lastStep.id, - toStepId: newStep.id, - priority: 0, - }, + await prisma.workflowTransition.createMany({ + data: this.defaultOutgoingTransitions(lastStep, newStep.id), }); } } @@ -589,6 +638,8 @@ export class WorkflowService { } } + await this.validateStepTemplate(projectId, data.templateId); + const updateData: Prisma.WorkflowStepUpdateInput = {}; if (data.name !== undefined) updateData.name = data.name; @@ -746,7 +797,7 @@ export class WorkflowService { /** * Splice a step out of the flow: re-wire its parent(s) directly to its child, - * then delete only the step itself. Not allowed for CONDITION or TRIGGER steps. + * then delete only the step itself. Not allowed for branching or TRIGGER steps. */ public static async spliceStep(projectId: string, workflowId: string, stepId: string): Promise { await this.get(projectId, workflowId); @@ -767,8 +818,8 @@ export class WorkflowService { throw new HttpException(400, 'Cannot remove the trigger step.'); } - if (step.type === 'CONDITION') { - throw new HttpException(400, 'Cannot splice a condition step out of the flow.'); + if (step.type === 'CONDITION' || step.type === 'WAIT_FOR_EVENT') { + throw new HttpException(400, 'Cannot splice a branching step out of the flow.'); } // Check for active executions on this step @@ -812,10 +863,8 @@ export class WorkflowService { * so a condition branch keeps its label) and a fresh transition carries the * flow on to the original target. * - * When the inserted step is itself a CONDITION, the downstream transition is - * attached to its first branch (`yes`) rather than left unconditional — a - * condition step routes exclusively by branch, so an unconditional outgoing - * transition would never be taken and B would be stranded. + * A newly inserted CONDITION starts on `yes`. A newly inserted WAIT_FOR_EVENT + * gets explicit event and, when configured, timeout routes to B. * * Purely additive — no step is removed and nothing becomes unreachable — so * unlike splice/delete this does not need to guard against in-flight executions. @@ -860,6 +909,8 @@ export class WorkflowService { ); } + await this.validateStepTemplate(projectId, data.templateId); + return prisma.$transaction(async tx => { const newStep = await tx.workflowStep.create({ data: { @@ -879,15 +930,8 @@ export class WorkflowService { data: {toStepId: newStep.id}, }); - await tx.workflowTransition.create({ - data: { - fromStepId: newStep.id, - toStepId: transition.toStepId, - // A freshly created condition has no config yet, which the builder reads - // as the default binary yes/no pair — so `yes` is its first branch. - condition: data.type === 'CONDITION' ? toPrismaJson({branch: 'yes'}) : Prisma.JsonNull, - priority: 0, - }, + await tx.workflowTransition.createMany({ + data: this.defaultOutgoingTransitions(newStep, transition.toStepId), }); return newStep; @@ -940,6 +984,7 @@ export class WorkflowService { fromStepId: string; toStepId: string; condition?: Prisma.JsonValue; + waitOutcome?: WorkflowWaitOutcome; priority?: number; }, ): Promise { @@ -967,32 +1012,61 @@ export class WorkflowService { throw new HttpException(400, 'An exit step ends the flow and cannot be connected to another step'); } - // For CONDITION steps, validate that this branch doesn't already have a transition - if (fromStep?.type === 'CONDITION' && data.condition) { - const conditionObj = - typeof data.condition === 'object' && data.condition !== null - ? (data.condition as Record) - : null; - if (conditionObj && 'branch' in conditionObj) { - // Check if a transition with this branch already exists - const existingTransition = await prisma.workflowTransition.findFirst({ - where: { - fromStepId: data.fromStepId, - condition: { - path: ['branch'], - equals: toPrismaJson(conditionObj.branch), + const hasCondition = data.condition !== undefined && data.condition !== null; + + if (fromStep?.type === 'WAIT_FOR_EVENT') { + if (hasCondition) { + throw new HttpException(400, 'Wait routes use waitOutcome, not transition conditions'); + } + if (!data.waitOutcome) { + throw new HttpException(400, 'WAIT_FOR_EVENT transitions require an EVENT or TIMEOUT outcome'); + } + if (data.waitOutcome === WorkflowWaitOutcome.TIMEOUT && !this.waitHasTimeout(fromStep.config)) { + throw new HttpException(400, 'A TIMEOUT transition requires a positive timeout on the wait step'); + } + + const existingOutcome = await prisma.workflowTransition.findFirst({ + where: {fromStepId: data.fromStepId, waitOutcome: data.waitOutcome}, + }); + if (existingOutcome) { + throw new HttpException(400, `The ${data.waitOutcome} route already has a transition`); + } + } else if (fromStep?.type === 'CONDITION') { + if (data.waitOutcome) { + throw new HttpException(400, 'Condition branches cannot declare a wait outcome'); + } + if (data.condition) { + const conditionObj = + typeof data.condition === 'object' && data.condition !== null + ? (data.condition as Record) + : null; + if (conditionObj && 'branch' in conditionObj) { + // Check if a transition with this branch already exists + const existingTransition = await prisma.workflowTransition.findFirst({ + where: { + fromStepId: data.fromStepId, + condition: { + path: ['branch'], + equals: toPrismaJson(conditionObj.branch), + }, }, - }, - }); + }); - if (existingTransition) { - throw new HttpException( - 400, - `A transition for the "${conditionObj.branch}" branch already exists from this step`, - ); + if (existingTransition) { + throw new HttpException( + 400, + `A transition for the "${conditionObj.branch}" branch already exists from this step`, + ); + } } } - } else if (fromStep?.type !== 'CONDITION') { + } else { + if (data.waitOutcome) { + throw new HttpException(400, 'Only WAIT_FOR_EVENT steps can declare a wait outcome'); + } + if (hasCondition) { + throw new HttpException(400, 'Only CONDITION steps can declare a transition condition'); + } // Every other step type routes to exactly one next step. A second outgoing // transition would make the next hop ambiguous at execution time. const existingOutgoing = await prisma.workflowTransition.findFirst({ @@ -1018,6 +1092,7 @@ export class WorkflowService { fromStepId: data.fromStepId, toStepId: data.toStepId, condition: data.condition ?? Prisma.JsonNull, + waitOutcome: data.waitOutcome, priority: data.priority ?? 0, }, }); diff --git a/apps/api/src/services/__tests__/SecurityService.sns.test.ts b/apps/api/src/services/__tests__/SecurityService.sns.test.ts new file mode 100644 index 000000000..26f5d10a0 --- /dev/null +++ b/apps/api/src/services/__tests__/SecurityService.sns.test.ts @@ -0,0 +1,75 @@ +import {createSign, generateKeyPairSync} from 'node:crypto'; + +import {afterEach, describe, expect, it, vi} from 'vitest'; + +import {SecurityService} from '../SecurityService'; + +const {ALLOWED_SNS_TOPICS} = vi.hoisted(() => ({ + ALLOWED_SNS_TOPICS: [ + 'arn:aws:sns:us-east-1:123456789012:plunk-ses-events', + 'arn:aws:sns:eu-west-1:123456789012:plunk-ses-inbound', + ] as const, +})); + +const {privateKey: snsPrivateKey, publicKey: snsPublicKey} = generateKeyPairSync('rsa', {modulusLength: 2048}); +const snsPublicKeyPem = snsPublicKey.export({type: 'spki', format: 'pem'}).toString(); + +vi.mock('../../app/constants.js', async () => { + const actual = await vi.importActual('../../app/constants.js'); + return { + ...actual, + SNS_TOPIC_ARNS: new Set(ALLOWED_SNS_TOPICS), + }; +}); + +function signedSnsNotification(topicArn: string, certName: string): Record { + const body = { + Type: 'Notification', + MessageId: `message-${certName}`, + TopicArn: topicArn, + Message: 'test message', + Timestamp: '2026-08-26T18:00:00.000Z', + SignatureVersion: '2', + SigningCertURL: `https://sns.us-east-1.amazonaws.com/${certName}.pem`, + } as Record; + const stringToSign = ['Message', 'MessageId', 'Timestamp', 'TopicArn', 'Type'] + .map(key => `${key}\n${body[key]}\n`) + .join(''); + + body.Signature = createSign('RSA-SHA256').update(stringToSign, 'utf8').sign(snsPrivateKey, 'base64'); + return body; +} + +describe('SecurityService SNS topic authorization', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it.each(ALLOWED_SNS_TOPICS)('accepts a valid signature from configured topic %s', async topicArn => { + const certName = topicArn.endsWith('inbound') ? 'allowed-inbound' : 'allowed-outbound'; + const fetchMock = vi.fn().mockResolvedValue(new Response(snsPublicKeyPem, {status: 200})); + vi.stubGlobal('fetch', fetchMock); + + await expect(SecurityService.verifySnsSignature(signedSnsNotification(topicArn, certName))).resolves.toBe(true); + expect(fetchMock).toHaveBeenCalledOnce(); + }); + + it('rejects a valid AWS signature from an unconfigured account before fetching its certificate', async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response(snsPublicKeyPem, {status: 200})); + vi.stubGlobal('fetch', fetchMock); + const untrusted = signedSnsNotification('arn:aws:sns:us-east-1:999999999999:plunk-ses-events', 'untrusted-account'); + + await expect(SecurityService.verifySnsSignature(untrusted)).resolves.toBe(false); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('rejects a missing TopicArn before fetching a signing certificate', async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response(snsPublicKeyPem, {status: 200})); + vi.stubGlobal('fetch', fetchMock); + const missingTopic = signedSnsNotification(ALLOWED_SNS_TOPICS[0], 'missing-topic'); + delete missingTopic.TopicArn; + + await expect(SecurityService.verifySnsSignature(missingTopic)).resolves.toBe(false); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/services/__tests__/WorkflowExecutionService.integration.test.ts b/apps/api/src/services/__tests__/WorkflowExecutionService.integration.test.ts index 5caedc15a..de3ee531f 100644 --- a/apps/api/src/services/__tests__/WorkflowExecutionService.integration.test.ts +++ b/apps/api/src/services/__tests__/WorkflowExecutionService.integration.test.ts @@ -1,5 +1,5 @@ import {beforeEach, describe, expect, it, vi} from 'vitest'; -import {StepExecutionStatus, WorkflowExecutionStatus, WorkflowStepType} from '@plunk/db'; +import {StepExecutionStatus, WorkflowExecutionStatus, WorkflowStepType, WorkflowWaitOutcome} from '@plunk/db'; import {toPrismaJson} from '@plunk/types'; import {WorkflowExecutionService} from '../WorkflowExecutionService'; import {factories, getPrismaClient} from '../../../../../test/helpers'; @@ -512,6 +512,173 @@ describe('WorkflowExecutionService - Integration Tests', () => { // Step should be completed after event arrives expect(waitStepExecution?.status).toBe(StepExecutionStatus.COMPLETED); }); + + it('should resume a persisted wait on the event route with merged event context', async () => { + const contact = await factories.createContact({projectId}); + const workflow = await factories.createWorkflow({projectId}); + const waitStep = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: WorkflowStepType.WAIT_FOR_EVENT, + name: 'Wait for Purchase', + position: {x: 100, y: 0}, + config: toPrismaJson({eventName: 'purchase.completed', timeout: 3600}), + }, + }); + const eventCondition = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: WorkflowStepType.CONDITION, + name: 'Inspect resumed plan', + position: {x: 200, y: -50}, + config: toPrismaJson({field: 'event.plan', operator: 'equals', value: 'pro'}), + }, + }); + const eventExit = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: WorkflowStepType.EXIT, + name: 'Event path', + position: {x: 300, y: -75}, + config: toPrismaJson({}), + }, + }); + const wrongPlanExit = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: WorkflowStepType.EXIT, + name: 'Wrong plan path', + position: {x: 300, y: -25}, + config: toPrismaJson({}), + }, + }); + const timeoutExit = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: WorkflowStepType.EXIT, + name: 'Timeout path', + position: {x: 200, y: 50}, + config: toPrismaJson({}), + }, + }); + + await prisma.workflowTransition.createMany({ + data: [ + {fromStepId: waitStep.id, toStepId: eventCondition.id, waitOutcome: WorkflowWaitOutcome.EVENT}, + {fromStepId: waitStep.id, toStepId: timeoutExit.id, waitOutcome: WorkflowWaitOutcome.TIMEOUT}, + {fromStepId: eventCondition.id, toStepId: eventExit.id, condition: toPrismaJson({branch: 'yes'})}, + {fromStepId: eventCondition.id, toStepId: wrongPlanExit.id, condition: toPrismaJson({branch: 'no'})}, + ], + }); + + // These durable rows are all a new worker needs after a restart mid-wait. + const execution = await prisma.workflowExecution.create({ + data: { + workflowId: workflow.id, + contactId: contact.id, + status: WorkflowExecutionStatus.WAITING, + currentStepId: waitStep.id, + context: toPrismaJson({source: 'signup', plan: 'free'}), + }, + }); + const waitExecution = await prisma.workflowStepExecution.create({ + data: { + executionId: execution.id, + stepId: waitStep.id, + status: StepExecutionStatus.WAITING, + startedAt: new Date(), + }, + }); + + await WorkflowExecutionService.handleEvent(projectId, 'purchase.completed', contact.id, { + plan: 'pro', + orderId: 'order-123', + }); + + const resumed = await prisma.workflowExecution.findUniqueOrThrow({where: {id: execution.id}}); + expect(resumed.context).toEqual({source: 'signup', plan: 'pro', orderId: 'order-123'}); + + const stepExecutions = await prisma.workflowStepExecution.findMany({ + where: {executionId: execution.id}, + include: {step: true}, + }); + expect(stepExecutions.some(item => item.step.id === eventExit.id)).toBe(true); + expect(stepExecutions.some(item => item.step.id === timeoutExit.id)).toBe(false); + expect(stepExecutions.some(item => item.step.id === wrongPlanExit.id)).toBe(false); + expect( + (stepExecutions.find(item => item.step.id === eventCondition.id)?.output as {branch?: string} | null)?.branch, + ).toBe('yes'); + expect( + (await prisma.workflowStepExecution.findUniqueOrThrow({where: {id: waitExecution.id}})).output, + ).toMatchObject({waitOutcome: WorkflowWaitOutcome.EVENT, eventData: {plan: 'pro', orderId: 'order-123'}}); + }); + + it('should follow only the timeout route when a persisted wait expires', async () => { + const contact = await factories.createContact({projectId}); + const workflow = await factories.createWorkflow({projectId}); + const waitStep = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: WorkflowStepType.WAIT_FOR_EVENT, + name: 'Wait for Purchase', + position: {x: 100, y: 0}, + config: toPrismaJson({eventName: 'purchase.completed', timeout: 3600}), + }, + }); + const eventExit = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: WorkflowStepType.EXIT, + name: 'Event path', + position: {x: 200, y: -50}, + config: toPrismaJson({}), + }, + }); + const timeoutExit = await prisma.workflowStep.create({ + data: { + workflowId: workflow.id, + type: WorkflowStepType.EXIT, + name: 'Timeout path', + position: {x: 200, y: 50}, + config: toPrismaJson({}), + }, + }); + await prisma.workflowTransition.createMany({ + data: [ + {fromStepId: waitStep.id, toStepId: eventExit.id, waitOutcome: WorkflowWaitOutcome.EVENT}, + {fromStepId: waitStep.id, toStepId: timeoutExit.id, waitOutcome: WorkflowWaitOutcome.TIMEOUT}, + ], + }); + const execution = await prisma.workflowExecution.create({ + data: { + workflowId: workflow.id, + contactId: contact.id, + status: WorkflowExecutionStatus.WAITING, + currentStepId: waitStep.id, + context: toPrismaJson({source: 'signup'}), + }, + }); + const waitExecution = await prisma.workflowStepExecution.create({ + data: { + executionId: execution.id, + stepId: waitStep.id, + status: StepExecutionStatus.WAITING, + startedAt: new Date(), + }, + }); + + await WorkflowExecutionService.processTimeout(execution.id, waitStep.id, waitExecution.id); + + const stepExecutions = await prisma.workflowStepExecution.findMany({ + where: {executionId: execution.id}, + select: {stepId: true}, + }); + expect(stepExecutions.some(item => item.stepId === timeoutExit.id)).toBe(true); + expect(stepExecutions.some(item => item.stepId === eventExit.id)).toBe(false); + expect( + (await prisma.workflowStepExecution.findUniqueOrThrow({where: {id: waitExecution.id}})).output, + ).toMatchObject({waitOutcome: WorkflowWaitOutcome.TIMEOUT, timedOut: true}); + }); }); // ======================================== diff --git a/apps/api/src/services/__tests__/WorkflowService.test.ts b/apps/api/src/services/__tests__/WorkflowService.test.ts index 412bb5c1f..ac6d10e67 100644 --- a/apps/api/src/services/__tests__/WorkflowService.test.ts +++ b/apps/api/src/services/__tests__/WorkflowService.test.ts @@ -1,5 +1,5 @@ import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; -import {WorkflowExecutionStatus, WorkflowStepType, WorkflowTriggerType} from '@plunk/db'; +import {WorkflowExecutionStatus, WorkflowStepType, WorkflowTriggerType, WorkflowWaitOutcome} from '@plunk/db'; import {WorkflowService} from '../WorkflowService'; import {Keys} from '../keys'; import {factories, getPrismaClient} from '../../../../../test/helpers'; @@ -426,9 +426,9 @@ describe('WorkflowService', () => { const mine = await factories.createWorkflow({projectId}); const foreign = await factories.createWorkflow({projectId: otherProject.id}); - await expect( - WorkflowService.bulkUpdate(projectId, {ids: [mine.id, foreign.id], delete: true}), - ).rejects.toThrow(/not found in this project/i); + await expect(WorkflowService.bulkUpdate(projectId, {ids: [mine.id, foreign.id], delete: true})).rejects.toThrow( + /not found in this project/i, + ); // Both survive — atomic rollback, no cross-project leak. expect(await prisma.workflow.findUnique({where: {id: mine.id}})).not.toBeNull(); @@ -443,9 +443,9 @@ describe('WorkflowService', () => { status: WorkflowExecutionStatus.RUNNING, }); - await expect( - WorkflowService.bulkUpdate(projectId, {ids: [busy.id, idle.id], delete: true}), - ).rejects.toThrow(/active execution/i); + await expect(WorkflowService.bulkUpdate(projectId, {ids: [busy.id, idle.id], delete: true})).rejects.toThrow( + /active execution/i, + ); // Partial delete must be impossible — the idle workflow must survive too. expect(await prisma.workflow.findUnique({where: {id: busy.id}})).not.toBeNull(); @@ -498,6 +498,44 @@ describe('WorkflowService', () => { expect(step.templateId).toBe(template.id); }); + it('should reject foreign and missing templates without revealing which exists', async () => { + const workflow = await factories.createWorkflow({projectId}); + const {project: otherProject} = await factories.createUserWithProject(); + const foreignTemplate = await factories.createTemplate({projectId: otherProject.id}); + const missingTemplateId = '00000000-0000-0000-0000-000000000000'; + + const addEmailStep = (templateId: string) => + WorkflowService.addStep(projectId, workflow.id, { + type: WorkflowStepType.SEND_EMAIL, + name: 'Send welcome email', + position: {x: 200, y: 100}, + config: {}, + templateId, + }); + + await expect(addEmailStep(foreignTemplate.id)).rejects.toMatchObject({ + code: 404, + message: 'Template not found', + }); + await expect(addEmailStep(missingTemplateId)).rejects.toMatchObject({ + code: 404, + message: 'Template not found', + }); + await expect( + WorkflowService.addStep(projectId, workflow.id, { + type: WorkflowStepType.DELAY, + name: 'Wait 1 hour', + position: {x: 200, y: 100}, + config: {delay: 3600}, + templateId: foreignTemplate.id, + }), + ).rejects.toMatchObject({code: 404, message: 'Template not found'}); + + const addedSteps = await prisma.workflowStep.findMany({where: {workflowId: workflow.id}}); + expect(addedSteps).toHaveLength(1); + expect(addedSteps[0]?.type).toBe(WorkflowStepType.TRIGGER); + }); + it('should auto-connect to previous step by default', async () => { const workflow = await factories.createWorkflow({projectId}); @@ -614,6 +652,33 @@ describe('WorkflowService', () => { expect(updated.templateId).toBe(template2.id); }); + it('should reject updating an email step to foreign and missing templates', async () => { + const workflow = await factories.createWorkflow({projectId}); + const template = await factories.createTemplate({projectId}); + const {project: otherProject} = await factories.createUserWithProject(); + const foreignTemplate = await factories.createTemplate({projectId: otherProject.id}); + const missingTemplateId = '00000000-0000-0000-0000-000000000000'; + const step = await factories.createWorkflowStep({ + workflowId: workflow.id, + type: WorkflowStepType.SEND_EMAIL, + templateId: template.id, + }); + + await expect( + WorkflowService.updateStep(projectId, workflow.id, step.id, { + templateId: foreignTemplate.id, + }), + ).rejects.toMatchObject({code: 404, message: 'Template not found'}); + await expect( + WorkflowService.updateStep(projectId, workflow.id, step.id, { + templateId: missingTemplateId, + }), + ).rejects.toMatchObject({code: 404, message: 'Template not found'}); + + const unchanged = await prisma.workflowStep.findUnique({where: {id: step.id}}); + expect(unchanged?.templateId).toBe(template.id); + }); + it('should remove template reference when set to null', async () => { const workflow = await factories.createWorkflow({projectId}); const template = await factories.createTemplate({projectId}); @@ -754,6 +819,83 @@ describe('WorkflowService', () => { expect(noTransition.condition).toEqual({branch: 'no'}); }); + it('should create distinct event and timeout routes from WAIT_FOR_EVENT steps', async () => { + const workflow = await factories.createWorkflow({projectId}); + const waitStep = await factories.createWorkflowStep({ + workflowId: workflow.id, + type: WorkflowStepType.WAIT_FOR_EVENT, + config: {eventName: 'purchase.completed', timeout: 3600}, + }); + const eventStep = await factories.createWorkflowStep({workflowId: workflow.id}); + const timeoutStep = await factories.createWorkflowStep({workflowId: workflow.id}); + + const eventTransition = await WorkflowService.createTransition(projectId, workflow.id, { + fromStepId: waitStep.id, + toStepId: eventStep.id, + waitOutcome: WorkflowWaitOutcome.EVENT, + }); + const timeoutTransition = await WorkflowService.createTransition(projectId, workflow.id, { + fromStepId: waitStep.id, + toStepId: timeoutStep.id, + waitOutcome: WorkflowWaitOutcome.TIMEOUT, + }); + + expect(eventTransition.waitOutcome).toBe(WorkflowWaitOutcome.EVENT); + expect(timeoutTransition.waitOutcome).toBe(WorkflowWaitOutcome.TIMEOUT); + expect(eventTransition.condition).toBeNull(); + expect(timeoutTransition.condition).toBeNull(); + }); + + it('should reject duplicate or condition-based WAIT_FOR_EVENT routes', async () => { + const workflow = await factories.createWorkflow({projectId}); + const waitStep = await factories.createWorkflowStep({ + workflowId: workflow.id, + type: WorkflowStepType.WAIT_FOR_EVENT, + }); + const step2 = await factories.createWorkflowStep({workflowId: workflow.id}); + const step3 = await factories.createWorkflowStep({workflowId: workflow.id}); + + await WorkflowService.createTransition(projectId, workflow.id, { + fromStepId: waitStep.id, + toStepId: step2.id, + waitOutcome: WorkflowWaitOutcome.EVENT, + }); + + await expect( + WorkflowService.createTransition(projectId, workflow.id, { + fromStepId: waitStep.id, + toStepId: step3.id, + waitOutcome: WorkflowWaitOutcome.EVENT, + }), + ).rejects.toThrow(/already has a transition/i); + + await expect( + WorkflowService.createTransition(projectId, workflow.id, { + fromStepId: waitStep.id, + toStepId: step3.id, + condition: {branch: 'timeout'}, + }), + ).rejects.toThrow(/waitOutcome/i); + }); + + it('should reject a timeout route when the wait has no timeout', async () => { + const workflow = await factories.createWorkflow({projectId}); + const waitStep = await factories.createWorkflowStep({ + workflowId: workflow.id, + type: WorkflowStepType.WAIT_FOR_EVENT, + config: {eventName: 'purchase.completed'}, + }); + const timeoutStep = await factories.createWorkflowStep({workflowId: workflow.id}); + + await expect( + WorkflowService.createTransition(projectId, workflow.id, { + fromStepId: waitStep.id, + toStepId: timeoutStep.id, + waitOutcome: WorkflowWaitOutcome.TIMEOUT, + }), + ).rejects.toThrow(/positive timeout/i); + }); + it('should reject a second outgoing transition from a non-condition step', async () => { const workflow = await factories.createWorkflow({projectId}); const step1 = await factories.createWorkflowStep({workflowId: workflow.id, type: WorkflowStepType.DELAY}); @@ -878,6 +1020,31 @@ describe('WorkflowService', () => { expect(outgoing[0]?.condition).toBeNull(); }); + it('should reject inserting an email step with a foreign template', async () => { + const workflow = await factories.createWorkflow({projectId}); + const stepA = await factories.createWorkflowStep({workflowId: workflow.id}); + const stepB = await factories.createWorkflowStep({workflowId: workflow.id}); + const transition = await prisma.workflowTransition.create({ + data: {fromStepId: stepA.id, toStepId: stepB.id}, + }); + const {project: otherProject} = await factories.createUserWithProject(); + const foreignTemplate = await factories.createTemplate({projectId: otherProject.id}); + const stepCount = await prisma.workflowStep.count({where: {workflowId: workflow.id}}); + + await expect( + WorkflowService.insertStepOnTransition(projectId, workflow.id, transition.id, { + type: WorkflowStepType.SEND_EMAIL, + name: 'Inserted email', + config: {}, + templateId: foreignTemplate.id, + }), + ).rejects.toMatchObject({code: 404, message: 'Template not found'}); + + const original = await prisma.workflowTransition.findUnique({where: {id: transition.id}}); + expect(original?.toStepId).toBe(stepB.id); + expect(await prisma.workflowStep.count({where: {workflowId: workflow.id}})).toBe(stepCount); + }); + it('should attach the downstream step to the first branch when inserting a CONDITION', async () => { const workflow = await factories.createWorkflow({projectId}); const stepA = await factories.createWorkflowStep({workflowId: workflow.id}); diff --git a/apps/web/src/components/WorkflowBuilder.tsx b/apps/web/src/components/WorkflowBuilder.tsx index e772b05c4..35cd4da00 100644 --- a/apps/web/src/components/WorkflowBuilder.tsx +++ b/apps/web/src/components/WorkflowBuilder.tsx @@ -51,12 +51,14 @@ interface WorkflowBuilderProps { id: string; toStepId: string; condition: unknown; + waitOutcome: 'EVENT' | 'TIMEOUT' | null; priority: number; }>; incomingTransitions: Array<{ id: string; fromStepId: string; condition: unknown; + waitOutcome: 'EVENT' | 'TIMEOUT' | null; priority: number; }>; })[]; @@ -156,6 +158,37 @@ function getBranchColor(config: any, branchId: string): string { return branchId === 'yes' ? '#16a34a' : '#dc2626'; } +type WaitOutcome = 'EVENT' | 'TIMEOUT'; +const WAIT_OUTCOMES: WaitOutcome[] = ['EVENT', 'TIMEOUT']; + +function waitHasTimeout(config: any): boolean { + return typeof config?.timeout === 'number' && config.timeout > 0; +} + +function getWaitOutcomeHandles(config: any, transitions: Array<{waitOutcome: WaitOutcome | null}> = []): WaitOutcome[] { + const expected = waitHasTimeout(config) ? WAIT_OUTCOMES : WAIT_OUTCOMES.slice(0, 1); + const wired = transitions.map(t => t.waitOutcome).filter((outcome): outcome is WaitOutcome => Boolean(outcome)); + + return [...expected, ...wired.filter(outcome => !expected.includes(outcome))]; +} + +function getWaitOutcomeLabel(outcome: WaitOutcome): string { + return outcome === 'EVENT' ? 'Event received' : 'Timed out'; +} + +function getWaitOutcomeColor(outcome: WaitOutcome): string { + return outcome === 'EVENT' ? '#16a34a' : '#d97706'; +} + +function getRoutePriority(stepType: string | undefined, config: any, outcome?: string | null): number { + if (!outcome) return 0; + + const routes = + stepType === 'WAIT_FOR_EVENT' ? WAIT_OUTCOMES : stepType === 'CONDITION' ? getExpectedBranches(config) : []; + const index = routes.findIndex(route => route === outcome); + return index >= 0 ? index : 0; +} + // Dagre layout function function getLayoutedElements(nodes: Node[], edges: Edge[]) { const dagreGraph = new dagre.graphlib.Graph(); @@ -289,9 +322,9 @@ function CustomNode({ template?: {id: string; name: string}; config?: any; /** - * One entry per outgoing slot. Condition steps get a handle per branch so a - * dragged connection carries the branch it started from; everything else has - * a single unnamed slot. Empty for EXIT, which ends the path. + * One entry per outgoing slot. Conditions and event waits get named handles + * so a dragged connection carries its route; linear steps get one unnamed + * slot. Empty for EXIT, which ends the path. */ sourceHandles?: {id: string; color: string; connectable: boolean}[]; }; @@ -305,12 +338,7 @@ function CustomNode({ return ( <> - +
{ const step = steps.find(s => s.id === stepId); - const isCondition = step?.type === 'CONDITION'; + const isBranching = step?.type === 'CONDITION' || step?.type === 'WAIT_FOR_EVENT'; const hasChildren = (step?.outgoingTransitions?.length ?? 0) > 0; - setDeleteMode(isCondition || !hasChildren ? 'cascade' : 'splice'); + setDeleteMode(isBranching || !hasChildren ? 'cascade' : 'splice'); setStepToDelete(stepId); setShowDeleteDialog(true); }, @@ -577,8 +605,8 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr const color = STEP_TYPE_COLORS[step.type as keyof typeof STEP_TYPE_COLORS] || '#6b7280'; const bgColor = STEP_TYPE_BG[step.type as keyof typeof STEP_TYPE_BG] || '#f3f4f6'; - // EXIT ends the path, a condition routes per branch, everything else has a - // single next step. + // EXIT ends the path. Conditions and event waits expose named routes; + // every other step has one unnamed next step. const outgoing = step.outgoingTransitions ?? []; const sourceHandles = step.type === 'EXIT' @@ -589,7 +617,13 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr color: getBranchColor(step.config, branchId), connectable: !outgoing.some(t => getTransitionBranch(t.condition) === branchId), })) - : [{id: SOURCE_HANDLE_ID, color: '#94a3b8', connectable: outgoing.length === 0}]; + : step.type === 'WAIT_FOR_EVENT' + ? getWaitOutcomeHandles(step.config, outgoing).map(outcome => ({ + id: outcome, + color: getWaitOutcomeColor(outcome), + connectable: !outgoing.some(t => t.waitOutcome === outcome), + })) + : [{id: SOURCE_HANDLE_ID, color: '#94a3b8', connectable: outgoing.length === 0}]; return { id: step.id, @@ -631,13 +665,28 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr draggable: false, data: { label: getBranchLabel(step.config, branchId), - onClick: () => setPickerContext({mode: 'append', fromStepId: step.id, branch: branchId}), + onClick: () => setPickerContext({mode: 'append', fromStepId: step.id, outcome: branchId}), + }, + }); + } + }); + } else if (step.type === 'WAIT_FOR_EVENT') { + getWaitOutcomeHandles(step.config, step.outgoingTransitions ?? []).forEach(outcome => { + if (!step.outgoingTransitions?.some(t => t.waitOutcome === outcome)) { + nodes.push({ + id: `${step.id}-add-${outcome}`, + type: 'addStep', + position: {x: 0, y: 0}, + draggable: false, + data: { + label: getWaitOutcomeLabel(outcome), + onClick: () => setPickerContext({mode: 'append', fromStepId: step.id, outcome}), }, }); } }); } else { - // For non-condition steps, add + node if no outgoing transitions + // For linear steps, add + node if no outgoing transitions. if (!step.outgoingTransitions || step.outgoingTransitions.length === 0) { nodes.push({ id: `${step.id}-add`, @@ -664,26 +713,34 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr if (step.outgoingTransitions && step.outgoingTransitions.length > 0) { step.outgoingTransitions.forEach(transition => { const condition = transition.condition; - const isConditional = condition && typeof condition === 'object' && 'branch' in condition; - const branch = + const conditionBranch = condition && typeof condition === 'object' && 'branch' in condition ? (condition.branch as string) : undefined; - - const branchColor = branch ? getBranchColor(step.config, branch) : '#94a3b8'; - const branchLabel = branch ? getBranchLabel(step.config, branch) : undefined; + const waitOutcome = step.type === 'WAIT_FOR_EVENT' ? transition.waitOutcome : undefined; + const route = conditionBranch ?? waitOutcome ?? undefined; + const routeColor = conditionBranch + ? getBranchColor(step.config, conditionBranch) + : waitOutcome + ? getWaitOutcomeColor(waitOutcome) + : '#64748b'; + const routeLabel = conditionBranch + ? getBranchLabel(step.config, conditionBranch) + : waitOutcome + ? getWaitOutcomeLabel(waitOutcome) + : undefined; edges.push({ id: transition.id, source: step.id, target: transition.toStepId, - sourceHandle: branch ?? SOURCE_HANDLE_ID, + sourceHandle: route ?? SOURCE_HANDLE_ID, targetHandle: TARGET_HANDLE_ID, type: 'workflow', animated: false, data: { - branchLabel: isConditional ? branchLabel : undefined, - branchColor: isConditional ? branchColor : '#64748b', + branchLabel: routeLabel, + branchColor: routeColor, onInsert: () => setPickerContext({mode: 'insert', transitionId: transition.id}), onDisconnect: () => setTransitionToDisconnect(transition.id), }, @@ -738,6 +795,30 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr }); } }); + } else if (step.type === 'WAIT_FOR_EVENT') { + getWaitOutcomeHandles(step.config, step.outgoingTransitions ?? []).forEach(outcome => { + if (!step.outgoingTransitions?.some(t => t.waitOutcome === outcome)) { + const color = getWaitOutcomeColor(outcome); + const label = getWaitOutcomeLabel(outcome); + + edges.push({ + id: `${step.id}-add-${outcome}-edge`, + source: step.id, + target: `${step.id}-add-${outcome}`, + sourceHandle: outcome, + targetHandle: TARGET_HANDLE_ID, + type: 'smoothstep', + animated: false, + label, + labelStyle: {fill: color, fontWeight: 600, fontSize: 12}, + labelBgStyle: {fill: '#fff', fillOpacity: 0.95}, + labelBgPadding: [8, 4] as [number, number], + labelBgBorderRadius: 4, + style: {stroke: '#94a3b8', strokeWidth: 2, strokeDasharray: '5,5'}, + markerEnd: {type: MarkerType.ArrowClosed, color: '#94a3b8', width: 20, height: 20}, + }); + } + }); } else { if (!step.outgoingTransitions || step.outgoingTransitions.length === 0) { edges.push({ @@ -767,7 +848,6 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr const [nodes, setNodes, onNodesChange] = useNodesState(layoutedNodes); const [edges, setEdges, onEdgesChange] = useEdgesState(layoutedEdges); - // Update nodes/edges when layout changes useEffect(() => { setNodes(layoutedNodes); @@ -816,31 +896,38 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr const addStepContext = pickerContext; try { - // Validate that this branch doesn't already have a transition + // Validate that this named route doesn't already have a transition. const fromStep = steps.find(s => s.id === addStepContext.fromStepId); if (!fromStep) { toast.error('Parent step not found'); return; } - // For CONDITION steps, check if the branch already exists - if (fromStep.type === 'CONDITION' && addStepContext.branch) { + if (fromStep.type === 'CONDITION' && addStepContext.outcome) { const existingBranchTransition = fromStep.outgoingTransitions?.find(t => { const condition = t.condition; return ( condition && typeof condition === 'object' && 'branch' in condition && - condition.branch === addStepContext.branch + condition.branch === addStepContext.outcome ); }); if (existingBranchTransition) { - toast.error(`The ${addStepContext.branch} branch already has a connection`); + toast.error(`The ${addStepContext.outcome} branch already has a connection`); return; } } + if ( + fromStep.type === 'WAIT_FOR_EVENT' && + addStepContext.outcome && + fromStep.outgoingTransitions?.some(t => t.waitOutcome === addStepContext.outcome) + ) { + toast.error(`The ${addStepContext.outcome} route already has a connection`); + return; + } - // Create the new step (autoConnect: false because we manually create the transition with branch info) + // Create the new step; the named route is connected explicitly below. const newStep = await network.fetch( 'POST', `/workflows/${workflowId}/steps`, @@ -849,17 +936,17 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr name: `New ${stepType.toLowerCase().replace('_', ' ')}`, position: {x: 0, y: 0}, // Will be auto-positioned by dagre layout config: {}, - autoConnect: false, // We manually create transitions to preserve branch information + autoConnect: false, }, ); const newStepId = newStep.id; - // Create the transition with proper condition - const condition = addStepContext.branch ? {branch: addStepContext.branch} : null; - const expectedBranches = fromStep.type === 'CONDITION' ? getExpectedBranches(fromStep.config) : []; - const branchIndex = addStepContext.branch ? expectedBranches.indexOf(addStepContext.branch) : -1; - const priority = branchIndex >= 0 ? branchIndex : 0; + const condition = + fromStep.type === 'CONDITION' && addStepContext.outcome ? {branch: addStepContext.outcome} : null; + const waitOutcome = + fromStep.type === 'WAIT_FOR_EVENT' ? (addStepContext.outcome as WaitOutcome | undefined) : undefined; + const priority = getRoutePriority(fromStep.type, fromStep.config, addStepContext.outcome); await network.fetch( 'POST', @@ -868,6 +955,7 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr fromStepId: addStepContext.fromStepId, toStepId: newStepId, condition, + waitOutcome, priority, }, ); @@ -1013,6 +1101,8 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr // The slot being dragged from has to be free if (fromStep.type === 'CONDITION') { if (fromStep.outgoingTransitions?.some(t => getTransitionBranch(t.condition) === sourceHandle)) return false; + } else if (fromStep.type === 'WAIT_FOR_EVENT') { + if (fromStep.outgoingTransitions?.some(t => t.waitOutcome === sourceHandle)) return false; } else if ((fromStep.outgoingTransitions?.length ?? 0) > 0) { return false; } @@ -1030,8 +1120,8 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr const fromStep = steps.find(s => s.id === source); const branch = fromStep?.type === 'CONDITION' ? sourceHandle : null; - const expectedBranches = fromStep?.type === 'CONDITION' ? getExpectedBranches(fromStep.config) : []; - const branchIndex = branch ? expectedBranches.indexOf(branch) : -1; + const waitOutcome = fromStep?.type === 'WAIT_FOR_EVENT' ? (sourceHandle as WaitOutcome | null) : null; + const priority = getRoutePriority(fromStep?.type, fromStep?.config, branch ?? waitOutcome); try { await network.fetch( @@ -1041,7 +1131,8 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr fromStepId: source, toStepId: target, condition: branch ? {branch} : null, - priority: branchIndex >= 0 ? branchIndex : 0, + waitOutcome: waitOutcome ?? undefined, + priority, }, ); @@ -1060,8 +1151,11 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr try { const fromStep = steps.find(s => s.id === pickerContext.fromStepId); - const expectedBranches = fromStep?.type === 'CONDITION' ? getExpectedBranches(fromStep.config) : []; - const branchIndex = pickerContext.branch ? expectedBranches.indexOf(pickerContext.branch) : -1; + const priority = getRoutePriority(fromStep?.type, fromStep?.config, pickerContext.outcome); + const condition = + fromStep?.type === 'CONDITION' && pickerContext.outcome ? {branch: pickerContext.outcome} : null; + const waitOutcome = + fromStep?.type === 'WAIT_FOR_EVENT' ? (pickerContext.outcome as WaitOutcome | undefined) : undefined; await network.fetch( 'POST', @@ -1069,8 +1163,9 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr { fromStepId: pickerContext.fromStepId, toStepId, - condition: pickerContext.branch ? {branch: pickerContext.branch} : null, - priority: branchIndex >= 0 ? branchIndex : 0, + condition, + waitOutcome, + priority, }, ); @@ -1084,7 +1179,6 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr [pickerContext, steps, workflowId, onUpdate], ); - if (steps.length === 0) { return (
@@ -1097,12 +1191,7 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr return ( <> - {isExpanded && ( -
setIsExpanded(false)} - /> - )} + {isExpanded &&
setIsExpanded(false)} />}
- + { const step = steps.find(s => s.id === node.id); @@ -1150,10 +1236,7 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr className="bg-white border border-neutral-200 rounded-lg shadow-md" maskColor="rgba(0, 0, 0, 0.05)" /> - +
@@ -1185,8 +1268,8 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr
- Click a + to add a step, hover a connection to insert or disconnect, or drag between the dots to - connect two steps. + Click a + to add a step, hover a connection to insert or disconnect, or drag between the dots to connect + two steps.
@@ -1314,9 +1397,10 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr const affectedSteps = getAffectedSteps(stepToDelete); const stepToDeleteData = steps.find(s => s.id === stepToDelete); const downstreamSteps = affectedSteps.filter(s => s.id !== stepToDelete); - const isCondition = stepToDeleteData?.type === 'CONDITION'; + const isBranching = + stepToDeleteData?.type === 'CONDITION' || stepToDeleteData?.type === 'WAIT_FOR_EVENT'; const hasChildren = downstreamSteps.length > 0; - const canSplice = !isCondition && hasChildren; + const canSplice = !isBranching && hasChildren; return ( @@ -1352,7 +1436,8 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr }`} >

- Delete with {downstreamSteps.length} downstream {downstreamSteps.length === 1 ? 'step' : 'steps'} + Delete with {downstreamSteps.length} downstream{' '} + {downstreamSteps.length === 1 ? 'step' : 'steps'}

Permanently removes this step and everything below it. This cannot be undone. @@ -1360,11 +1445,11 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr

- ) : isCondition && hasChildren ? ( + ) : isBranching && hasChildren ? (

- Removing a condition step will also delete all {downstreamSteps.length} downstream{' '} - {downstreamSteps.length === 1 ? 'step' : 'steps'} across its branches: + Removing this branching step will also delete all {downstreamSteps.length} downstream{' '} + {downstreamSteps.length === 1 ? 'step' : 'steps'} across its routes:

    {downstreamSteps.map(step => ( @@ -1376,7 +1461,9 @@ export function WorkflowBuilder({workflowId, steps, onUpdate}: WorkflowBuilderPr

    This action cannot be undone.

) : ( -

This step is removed from the workflow. This can't be undone.

+

+ This step is removed from the workflow. This can't be undone. +

)} diff --git a/apps/web/src/components/WorkflowVisualizer.tsx b/apps/web/src/components/WorkflowVisualizer.tsx index ee61c5c8e..e051ec2ae 100644 --- a/apps/web/src/components/WorkflowVisualizer.tsx +++ b/apps/web/src/components/WorkflowVisualizer.tsx @@ -26,12 +26,14 @@ interface WorkflowVisualizerProps { id: string; toStepId: string; condition: unknown; + waitOutcome: 'EVENT' | 'TIMEOUT' | null; priority: number; }>; incomingTransitions: Array<{ id: string; fromStepId: string; condition: unknown; + waitOutcome: 'EVENT' | 'TIMEOUT' | null; priority: number; }>; })[]; @@ -325,6 +327,27 @@ export function WorkflowVisualizer({steps}: WorkflowVisualizerProps) { transition.condition && typeof transition.condition === 'object' && 'branch' in transition.condition ? transition.condition.branch : undefined; + const waitOutcome = step.type === 'WAIT_FOR_EVENT' ? transition.waitOutcome : null; + const routeLabel = isConditional + ? branch === 'yes' + ? 'Yes' + : 'No' + : waitOutcome === 'EVENT' + ? 'Event received' + : waitOutcome === 'TIMEOUT' + ? 'Timed out' + : undefined; + const routeColor = isConditional + ? branch === 'yes' + ? '#16a34a' + : branch === 'no' + ? '#dc2626' + : '#64748b' + : waitOutcome === 'EVENT' + ? '#16a34a' + : waitOutcome === 'TIMEOUT' + ? '#d97706' + : '#64748b'; edges.push({ id: transition.id, @@ -332,9 +355,9 @@ export function WorkflowVisualizer({steps}: WorkflowVisualizerProps) { target: transition.toStepId, type: 'smoothstep', animated: false, - label: isConditional ? (branch === 'yes' ? 'Yes' : 'No') : undefined, + label: routeLabel, labelStyle: { - fill: branch === 'yes' ? '#16a34a' : branch === 'no' ? '#dc2626' : '#64748b', + fill: routeColor, fontWeight: 600, fontSize: 12, }, diff --git a/apps/web/src/pages/workflows/[id].tsx b/apps/web/src/pages/workflows/[id].tsx index 2d4a2735e..4029d9a29 100644 --- a/apps/web/src/pages/workflows/[id].tsx +++ b/apps/web/src/pages/workflows/[id].tsx @@ -273,6 +273,23 @@ export default function WorkflowEditorPage() { } } } + + if (step.type === 'WAIT_FOR_EVENT' && step.outgoingTransitions) { + const config = step.config && typeof step.config === 'object' && !Array.isArray(step.config) ? step.config : {}; + const expectedOutcomes = + typeof config.timeout === 'number' && config.timeout > 0 ? ['EVENT', 'TIMEOUT'] : ['EVENT']; + const missingOutcomes = expectedOutcomes.filter( + outcome => !step.outgoingTransitions.some(transition => transition.waitOutcome === outcome), + ); + + if (missingOutcomes.length > 0) { + errors.push( + `"${step.name}" wait step is missing connections for: ${missingOutcomes + .map(outcome => (outcome === 'EVENT' ? 'Event received' : 'Timed out')) + .join(', ')}`, + ); + } + } }); return {valid: errors.length === 0, errors}; @@ -893,4 +910,3 @@ function SettingsDialog({workflow, open, onOpenChange, onSave}: SettingsDialogPr ); } - diff --git a/apps/wiki/content/docs/api-reference/overview.mdx b/apps/wiki/content/docs/api-reference/overview.mdx index c37e4f9ad..ffc0c0fa0 100644 --- a/apps/wiki/content/docs/api-reference/overview.mdx +++ b/apps/wiki/content/docs/api-reference/overview.mdx @@ -291,7 +291,7 @@ Each workflow consists of a **workflow** record, a graph of **steps**, **transit | PATCH | `/workflows/:id/steps/:stepId` | Update a step's config. | | DELETE | `/workflows/:id/steps/:stepId?splice=true` | Delete a step. Pass `splice=true` to auto-reconnect surrounding transitions. | | **Transitions** | | | -| POST | `/workflows/:id/transitions` | Add a transition between two steps. For `CONDITION` steps, include `branch: "yes" \| "no"`. | +| POST | `/workflows/:id/transitions` | Add a transition. Use `condition.branch` for a `CONDITION` route or `waitOutcome: "EVENT" \| "TIMEOUT"` for a `WAIT_FOR_EVENT` route. | | DELETE | `/workflows/:id/transitions/:transitionId` | Delete a transition. | | **Executions** | | | | POST | `/workflows/:id/executions` | Manually start an execution for a contact. Optional `context` JSON for per-execution variables. | diff --git a/apps/wiki/content/docs/concepts/workflows.mdx b/apps/wiki/content/docs/concepts/workflows.mdx index 3dc68ff20..2dbeb811c 100644 --- a/apps/wiki/content/docs/concepts/workflows.mdx +++ b/apps/wiki/content/docs/concepts/workflows.mdx @@ -35,7 +35,7 @@ A workflow always begins with a single auto-created `TRIGGER` step. You build th | `TRIGGER` | Auto-created entry point. Holds the trigger configuration. | `eventName` (for `EVENT` trigger) | | `SEND_EMAIL` | Sends an email to the contact using a template. Template variables resolve from contact data + execution context. | `templateId`, optional `from` override | | `DELAY` | Pauses the execution for a fixed duration before continuing. | `amount`, `unit` (`minutes` / `hours` / `days`) | -| `WAIT_FOR_EVENT` | Pauses until a specified event is tracked on the contact, with a timeout fallback. | `eventName`, `timeout` (seconds) | +| `WAIT_FOR_EVENT` | Pauses until a specified event is tracked on the contact. Event arrival and timeout have distinct outgoing routes. | `eventName`, optional `timeout` (seconds) | | `CONDITION` | Branches the execution based on contact data or event data. Each `CONDITION` step has two outgoing transitions tagged `yes` / `no`. | A filter expression (same shape as segment filters) | | `WEBHOOK` | Calls an external HTTPS endpoint with contact + execution context as the JSON body. `url`, header values, and `body` support `{{variables}}`. | `url`, optional `method`, `headers`, `body` | | `UPDATE_CONTACT` | Patches contact data — useful for tagging contacts as they progress (`{ stage: "activated" }`). | `data` object | @@ -43,9 +43,9 @@ A workflow always begins with a single auto-created `TRIGGER` step. You build th ## Transitions -Transitions are the edges of the workflow graph. For most step types a transition is a simple "next" pointer. For `CONDITION` steps, each transition carries a `branch` discriminator (`"yes"` or `"no"`) so the engine knows which path to take when the condition evaluates. +Transitions are the edges of the workflow graph. For most step types a transition is a simple "next" pointer. `CONDITION` transitions carry their branch in `condition`; `WAIT_FOR_EVENT` transitions use a separate `waitOutcome` of `EVENT` or `TIMEOUT`. A timeout route is available only when the wait step has a positive timeout. -When deleting a step in the middle of a chain, pass `?splice=true` on `DELETE /workflows/:id/steps/:stepId` to automatically reconnect the surrounding transitions; otherwise the deletion leaves the graph disconnected. +When deleting a non-branching step in the middle of a chain, pass `?splice=true` on `DELETE /workflows/:id/steps/:stepId` to automatically reconnect the surrounding transitions; otherwise the deletion leaves the graph disconnected. Condition and event-wait steps must be removed with their branches rather than spliced. ## Executions @@ -60,7 +60,7 @@ Each contact entering the workflow creates a `WorkflowExecution`. Executions mov | `FAILED` | An unrecoverable error occurred (e.g. webhook returned non-2xx after retries, template not found). | | `CANCELLED` | Cancelled manually via the API or dashboard. | -Each execution carries a `context` JSON object that's merged with the contact's `data` when rendering templates and evaluating conditions. When you start an execution manually with `POST /workflows/:id/executions`, you can pass an initial `context` — this is how you parameterise per-execution variables (a coupon code, a referrer name) without storing them on the contact. +Each execution carries a `context` JSON object that's merged with the contact's `data` when rendering templates and evaluating conditions. When you start an execution manually with `POST /workflows/:id/executions`, you can pass an initial `context` — this is how you parameterise per-execution variables (a coupon code, a referrer name) without storing them on the contact. When a `WAIT_FOR_EVENT` resumes, the arriving event data is shallow-merged into that context, replacing keys with the newer event values. Downstream email, condition, and webhook steps can read it through `event.*`. ## Locking active workflows diff --git a/apps/wiki/content/docs/guides/idempotency.mdx b/apps/wiki/content/docs/guides/idempotency.mdx index f097163ab..f6419a566 100644 --- a/apps/wiki/content/docs/guides/idempotency.mdx +++ b/apps/wiki/content/docs/guides/idempotency.mdx @@ -41,7 +41,7 @@ curl -X POST https://api.useplunk.com/v1/send \ await fetch('https://api.useplunk.com/v1/send', { method: 'POST', headers: { - Authorization: 'Bearer sk_your_secret_key', + 'Authorization': 'Bearer sk_your_secret_key', 'Idempotency-Key': 'receipt-order-1234', 'Content-Type': 'application/json', }, @@ -100,21 +100,29 @@ Reusing a key returns `409` with the error code `IDEMPOTENCY_KEY_REUSED`. The `d An `originalStatusCode` of `null` means the original request is still in flight — two requests with the same key arrived at once, and this one lost the race. -Plunk **refuses** a reused key; it does not replay the original response. A `409` tells you the operation was not performed twice, but it does not return the original email or event ID. If you need that ID, record it when the first request succeeds. + Plunk **refuses** a reused key; it does not replay the original response. A `409` tells you the operation was not + performed twice, but it does not return the original email or event ID. If you need that ID, record it when the first + request succeeds. ## Which failures free the key Not every failed request burns the key. -| Outcome of the first request | Key is | Why | -| ---------------------------- | ------ | --- | -| `2xx` success | **Kept** | The operation happened. A retry would duplicate it. | -| `4xx` client error | **Released** | Validation and permission errors are rejected before anything is written, so it is safe to fix the request and retry with the same key. | -| `5xx` server error | **Kept** | The request may have partially completed. Refusing the retry is the whole point of the key. | +| Outcome of the first request | Key is | Why | +| ---------------------------- | ------------ | --------------------------------------------------------------------------------------------------------------------------------------- | +| `2xx` success | **Kept** | The operation happened. A retry would duplicate it. | +| `4xx` client error | **Released** | Validation and permission errors are rejected before anything is written, so it is safe to fix the request and retry with the same key. | +| `5xx` server error | **Kept** | The request may have partially completed. Refusing the retry is the whole point of the key. | If a `5xx` burns a key for a request you're confident never went through, retry with a new key. +`POST /v1/track` returns `2xx` as soon as the event is stored. Workflow dispatch +usually finishes in the request, but Plunk retries it internally with bounded +backoff if needed. An expiring dispatch lease prevents recovery from overlapping +active work after a process restart. The event ID in the response identifies the +accepted event; do not submit the same logical event again with a new key. + ## Scope and expiry Keys are scoped to your **project**, not to an endpoint. A key used on `/v1/track` cannot be reused on `/v1/send`. Two different projects can use the same key string independently. diff --git a/apps/wiki/content/docs/self-hosting/email-setup.mdx b/apps/wiki/content/docs/self-hosting/email-setup.mdx index 0756b012b..560114ced 100644 --- a/apps/wiki/content/docs/self-hosting/email-setup.mdx +++ b/apps/wiki/content/docs/self-hosting/email-setup.mdx @@ -39,10 +39,11 @@ description: Configure email delivery 2. Type: Standard 3. Name: `plunk-ses-events` 4. Create topic -5. Create subscription: +5. Copy the topic ARN. You will use it as `SNS_TOPIC_ARNS` in step 4. +6. Create subscription: - Protocol: HTTPS - Endpoint: `https://api.yourdomain.com/webhooks/sns` -6. Plunk automatically confirms the subscription. If it fails, check your logs for the confirmation URL. +7. Plunk automatically confirms the subscription. If it fails, check your logs for the confirmation URL. ## 3. Create Configuration Sets @@ -66,6 +67,7 @@ description: Configure email delivery AWS_SES_REGION="us-east-1" AWS_SES_ACCESS_KEY_ID="your-access-key" AWS_SES_SECRET_ACCESS_KEY="your-secret-key" +SNS_TOPIC_ARNS="arn:aws:sns:us-east-1:123456789012:plunk-ses-events" SES_CONFIGURATION_SET="plunk-tracking" SES_CONFIGURATION_SET_NO_TRACKING="plunk-no-tracking" ``` @@ -88,6 +90,8 @@ Not all AWS regions support SES inbound — confirm `inbound-smtp.. - **Actions**: Publish to Amazon SNS topic → select your `plunk-ses-events` topic (or create a separate topic that's also subscribed to `https://api.yourdomain.com/webhooks/sns`) 3. Set the rule set as **active** +If inbound email uses a separate topic, append its exact ARN to `SNS_TOPIC_ARNS`, separated by a comma. + ### Inbound IAM permissions Plunk doesn't call SES inbound APIs at runtime — the IAM policy from step 1 covers everything Plunk needs. Configuring receipt rules in AWS is a manual one-time action you do as an AWS admin. @@ -119,4 +123,3 @@ After approval, your sending quota will reflect a much higher daily and per-seco ## 8. (Optional) Configure a MAIL FROM Domain For better DMARC alignment, you can configure a custom MAIL FROM subdomain (e.g. `mail.yourdomain.com`). In the SES console under **Verified identities** → your domain → **MAIL FROM domain**, set a subdomain and add the additional MX and TXT records SES displays. Plunk's IAM policy already includes `ses:SetIdentityMailFromDomain` to support this. - diff --git a/apps/wiki/content/docs/self-hosting/environment-variables.mdx b/apps/wiki/content/docs/self-hosting/environment-variables.mdx index ebcc3b5da..e3ed59c4e 100644 --- a/apps/wiki/content/docs/self-hosting/environment-variables.mdx +++ b/apps/wiki/content/docs/self-hosting/environment-variables.mdx @@ -34,6 +34,7 @@ Set your subdomains here. The application automatically derives all internal and | `AWS_SES_REGION` | Yes | AWS region where SES is configured. | `us-east-1` | | `AWS_SES_ACCESS_KEY_ID` | Yes | AWS access key ID with SES send permissions. | `AKIA...` | | `AWS_SES_SECRET_ACCESS_KEY` | Yes | AWS secret access key for SES. | `wJalr...` | +| `SNS_TOPIC_ARNS` | Yes | Comma-separated exact SNS topic ARNs authorized to deliver SES events. Include each outbound and inbound topic subscribed to `/webhooks/sns`. | `arn:aws:sns:us-east-1:123456789012:plunk-ses-events` | | `SES_CONFIGURATION_SET` | No | SES configuration set name used for open/click tracking. | `plunk-configuration-set` (default) | | `SES_CONFIGURATION_SET_NO_TRACKING` | No | A second SES configuration set without tracking. When set, projects can toggle email tracking on/off. If omitted, the tracking toggle is hidden. | `plunk-no-tracking-configuration-set` (default) | | `MAIL_FROM_SUBDOMAIN` | No | Subdomain prefix used when constructing the MAIL FROM hostname for a verified domain (e.g. with default `plunk` and domain `yourdomain.com`, the MAIL FROM is `plunk.yourdomain.com`). Override when the default subdomain is already in use (e.g. by an R2/CDN custom domain), since the MAIL FROM hostname needs MX + TXT records that can't coexist with a CNAME. | `plunk` | diff --git a/docker-compose.yml b/docker-compose.yml index 18c0524fc..0546ae951 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -151,6 +151,7 @@ services: AWS_SES_REGION: ${AWS_SES_REGION} AWS_SES_ACCESS_KEY_ID: ${AWS_SES_ACCESS_KEY_ID} AWS_SES_SECRET_ACCESS_KEY: ${AWS_SES_SECRET_ACCESS_KEY} + SNS_TOPIC_ARNS: ${SNS_TOPIC_ARNS} SES_CONFIGURATION_SET: ${SES_CONFIGURATION_SET} SES_CONFIGURATION_SET_NO_TRACKING: ${SES_CONFIGURATION_SET_NO_TRACKING:-} diff --git a/packages/db/prisma/migrations/20260826120000_add_sns_webhook_receipts/migration.sql b/packages/db/prisma/migrations/20260826120000_add_sns_webhook_receipts/migration.sql new file mode 100644 index 000000000..ed04427a7 --- /dev/null +++ b/packages/db/prisma/migrations/20260826120000_add_sns_webhook_receipts/migration.sql @@ -0,0 +1,33 @@ +-- Durable SNS delivery receipts. The unique MessageId claim serializes concurrent +-- deliveries, while FAILED and stale PROCESSING rows remain reclaimable after a +-- handler or process failure. + +-- CreateEnum +CREATE TYPE "SnsWebhookReceiptStatus" AS ENUM ('PROCESSING', 'COMPLETED', 'FAILED'); + +-- CreateTable +CREATE TABLE "sns_webhook_receipts" ( + "id" TEXT NOT NULL, + "messageId" TEXT NOT NULL, + "status" "SnsWebhookReceiptStatus" NOT NULL DEFAULT 'PROCESSING', + "processingToken" TEXT NOT NULL, + "processingStartedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "completedAt" TIMESTAMP(3), + "expiresAt" TIMESTAMP(3) NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "sns_webhook_receipts_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "sns_webhook_receipts_messageId_key" ON "sns_webhook_receipts"("messageId"); + +-- CreateIndex +CREATE INDEX "sns_webhook_receipts_expiresAt_idx" ON "sns_webhook_receipts"("expiresAt"); + +-- Make event ingestion durable before workflow dispatch. Existing events have +-- already passed through the synchronous dispatcher and must not be replayed. +ALTER TABLE "events" ADD COLUMN "processedAt" TIMESTAMP(3); +UPDATE "events" SET "processedAt" = "createdAt"; +CREATE INDEX "events_processedAt_createdAt_idx" ON "events"("processedAt", "createdAt"); diff --git a/packages/db/prisma/migrations/20260826213000_add_workflow_wait_outcomes/migration.sql b/packages/db/prisma/migrations/20260826213000_add_workflow_wait_outcomes/migration.sql new file mode 100644 index 000000000..0b5a1edf9 --- /dev/null +++ b/packages/db/prisma/migrations/20260826213000_add_workflow_wait_outcomes/migration.sql @@ -0,0 +1,59 @@ +-- CreateEnum +CREATE TYPE "WorkflowWaitOutcome" AS ENUM ('EVENT', 'TIMEOUT'); + +-- AlterTable +ALTER TABLE "workflow_transitions" ADD COLUMN "waitOutcome" "WorkflowWaitOutcome"; + +-- Existing WAIT_FOR_EVENT transitions were linear: the same next step handled +-- both an arriving event and a configured timeout. Preserve that behavior while +-- moving the route out of the generic condition JSON. +UPDATE "workflow_transitions" AS transition +SET + "waitOutcome" = CASE + WHEN transition."condition"->>'branch' = 'timeout' + OR transition."condition"->>'fallback' = 'true' + THEN 'TIMEOUT'::"WorkflowWaitOutcome" + ELSE 'EVENT'::"WorkflowWaitOutcome" + END, + "condition" = NULL +FROM "workflow_steps" AS step +WHERE transition."fromStepId" = step."id" + AND step."type" = 'WAIT_FOR_EVENT'; + +-- A legacy unconditional wait transition was taken for either outcome. When a +-- timeout is configured, duplicate its target as the explicit timeout route. +INSERT INTO "workflow_transitions" ( + "id", + "fromStepId", + "toStepId", + "condition", + "waitOutcome", + "priority", + "createdAt", + "updatedAt" +) +SELECT + gen_random_uuid()::text, + transition."fromStepId", + transition."toStepId", + NULL, + 'TIMEOUT'::"WorkflowWaitOutcome", + transition."priority" + 1, + transition."createdAt", + transition."updatedAt" +FROM "workflow_transitions" AS transition +JOIN "workflow_steps" AS step ON step."id" = transition."fromStepId" +WHERE step."type" = 'WAIT_FOR_EVENT' + AND transition."waitOutcome" = 'EVENT' + AND jsonb_typeof(step."config"->'timeout') = 'number' + AND (step."config"->>'timeout')::double precision > 0 + AND NOT EXISTS ( + SELECT 1 + FROM "workflow_transitions" AS timeout_transition + WHERE timeout_transition."fromStepId" = transition."fromStepId" + AND timeout_transition."waitOutcome" = 'TIMEOUT' + ); + +-- CreateIndex +CREATE UNIQUE INDEX "workflow_transitions_fromStepId_waitOutcome_key" +ON "workflow_transitions"("fromStepId", "waitOutcome"); diff --git a/packages/db/prisma/migrations/20260827120000_link_workflow_execution_source_event/migration.sql b/packages/db/prisma/migrations/20260827120000_link_workflow_execution_source_event/migration.sql new file mode 100644 index 000000000..b928a2271 --- /dev/null +++ b/packages/db/prisma/migrations/20260827120000_link_workflow_execution_source_event/migration.sql @@ -0,0 +1,26 @@ +-- Link event-triggered workflow executions to the durable source event. The +-- reconciliation sweep uses this identity to recognize completed enrollment +-- without replaying workflow side effects. + +ALTER TABLE "workflow_executions" ADD COLUMN "sourceEventId" TEXT; +ALTER TABLE "workflow_step_executions" ADD COLUMN "resumeEventId" TEXT; + +ALTER TABLE "events" +ADD COLUMN "dispatchAttempts" INTEGER NOT NULL DEFAULT 0, +ADD COLUMN "nextDispatchAt" TIMESTAMP(3), +ADD COLUMN "dispatchFailedAt" TIMESTAMP(3), +ADD COLUMN "dispatchError" TEXT, +ADD COLUMN "dispatchLeaseId" TEXT, +ADD COLUMN "dispatchLeaseExpiresAt" TIMESTAMP(3); + +ALTER TABLE "workflow_executions" +ADD CONSTRAINT "workflow_executions_sourceEventId_fkey" +FOREIGN KEY ("sourceEventId") REFERENCES "events"("id") +ON DELETE SET NULL ON UPDATE CASCADE +NOT VALID; + +ALTER TABLE "workflow_step_executions" +ADD CONSTRAINT "workflow_step_executions_resumeEventId_fkey" +FOREIGN KEY ("resumeEventId") REFERENCES "events"("id") +ON DELETE SET NULL ON UPDATE CASCADE +NOT VALID; diff --git a/packages/db/prisma/migrations/20260827120100_validate_workflow_execution_source_event/migration.sql b/packages/db/prisma/migrations/20260827120100_validate_workflow_execution_source_event/migration.sql new file mode 100644 index 000000000..d269916db --- /dev/null +++ b/packages/db/prisma/migrations/20260827120100_validate_workflow_execution_source_event/migration.sql @@ -0,0 +1,8 @@ +-- Validation uses a lock that permits normal reads and writes. Keeping it +-- separate from constraint creation avoids a blocking validation scan while +-- the foreign key is installed. +ALTER TABLE "workflow_executions" +VALIDATE CONSTRAINT "workflow_executions_sourceEventId_fkey"; + +ALTER TABLE "workflow_step_executions" +VALIDATE CONSTRAINT "workflow_step_executions_resumeEventId_fkey"; diff --git a/packages/db/prisma/migrations/20260827120200_index_workflow_execution_source_event/migration.sql b/packages/db/prisma/migrations/20260827120200_index_workflow_execution_source_event/migration.sql new file mode 100644 index 000000000..3285e920b --- /dev/null +++ b/packages/db/prisma/migrations/20260827120200_index_workflow_execution_source_event/migration.sql @@ -0,0 +1,4 @@ +-- This table may already hold millions of executions. CONCURRENTLY keeps +-- normal reads and writes available while PostgreSQL builds the relation index. +CREATE INDEX CONCURRENTLY "workflow_executions_sourceEventId_idx" +ON "workflow_executions"("sourceEventId"); diff --git a/packages/db/prisma/migrations/20260827120300_unique_workflow_execution_source_event/migration.sql b/packages/db/prisma/migrations/20260827120300_unique_workflow_execution_source_event/migration.sql new file mode 100644 index 000000000..5aa1d8ff8 --- /dev/null +++ b/packages/db/prisma/migrations/20260827120300_unique_workflow_execution_source_event/migration.sql @@ -0,0 +1,4 @@ +-- The unique index is the concurrency guard for duplicate enrollment from one +-- event. Build it without blocking workflow-execution writes during deploy. +CREATE UNIQUE INDEX CONCURRENTLY "workflow_executions_workflowId_sourceEventId_key" +ON "workflow_executions"("workflowId", "sourceEventId"); diff --git a/packages/db/prisma/migrations/20260827120400_index_pending_event_dispatch/migration.sql b/packages/db/prisma/migrations/20260827120400_index_pending_event_dispatch/migration.sql new file mode 100644 index 000000000..de303601e --- /dev/null +++ b/packages/db/prisma/migrations/20260827120400_index_pending_event_dispatch/migration.sql @@ -0,0 +1,6 @@ +-- Only undelivered, retryable events participate in reconciliation. Excluding +-- processed history keeps this online-built index small and focused on the +-- maintenance worker's query. +CREATE INDEX CONCURRENTLY "events_dispatch_reconciliation_idx" +ON "events"("dispatchLeaseExpiresAt", "nextDispatchAt", "createdAt") +WHERE "processedAt" IS NULL AND "dispatchFailedAt" IS NULL; diff --git a/packages/db/prisma/migrations/20260827120500_index_workflow_step_resume_event/migration.sql b/packages/db/prisma/migrations/20260827120500_index_workflow_step_resume_event/migration.sql new file mode 100644 index 000000000..2dd2d5f03 --- /dev/null +++ b/packages/db/prisma/migrations/20260827120500_index_workflow_step_resume_event/migration.sql @@ -0,0 +1,4 @@ +-- A failed continuation is looked up by its source event on every retry. Build +-- that relation index without blocking workflow-step writes during deploy. +CREATE INDEX CONCURRENTLY "workflow_step_executions_resumeEventId_idx" +ON "workflow_step_executions"("resumeEventId"); diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 157efb7f6..57543b769 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -53,11 +53,11 @@ model Project { // Card verification: the onboarding charge is customer-initiated and on-session, which // proves nothing about whether the card accepts the merchant-initiated renewal a month // later. A second, off-session charge tests that directly. See CardVerificationService. - cardVerification CardVerificationStatus? - cardVerificationIntent String? // PaymentIntent of the last attempt, for support/debugging - cardVerificationAt DateTime? // When the current status was set; drives the reconciliation sweep - cardVerificationSession String? // Checkout session, so a retry reuses the same Stripe idempotency key - cardVerificationAttempts Int @default(0) // Sweep retries spent; bounds them and marks escalation + cardVerification CardVerificationStatus? + cardVerificationIntent String? // PaymentIntent of the last attempt, for support/debugging + cardVerificationAt DateTime? // When the current status was set; drives the reconciliation sweep + cardVerificationSession String? // Checkout session, so a retry reuses the same Stripe idempotency key + cardVerificationAttempts Int @default(0) // Sweep retries spent; bounds them and marks escalation // Billing Limits (per calendar month, null = unlimited) billingLimitWorkflows Int? // Max workflow emails per month @@ -438,13 +438,15 @@ model WorkflowTransition { toStepId String // Conditional routing - condition Json? // null = always follow, or { branch: "yes" } for condition steps - priority Int @default(0) // Order to evaluate transitions + condition Json? // CONDITION steps: { branch: "yes" }; null for other step types + waitOutcome WorkflowWaitOutcome? // WAIT_FOR_EVENT steps: event arrival or timeout + priority Int @default(0) // Order to evaluate transitions // Timestamps createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + @@unique([fromStepId, waitOutcome]) @@index([fromStepId]) @@index([toStepId]) @@map("workflow_transitions") @@ -460,6 +462,11 @@ model WorkflowExecution { contact Contact @relation(fields: [contactId], references: [id], onDelete: Cascade) contactId String + // Event that enrolled this execution. The durable event outbox uses this to + // recognize completed enrollment without replaying workflow side effects. + sourceEvent Event? @relation(fields: [sourceEventId], references: [id], onDelete: SetNull) + sourceEventId String? + // Execution state status WorkflowExecutionStatus @default(RUNNING) @@ -484,10 +491,12 @@ model WorkflowExecution { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + @@unique([workflowId, sourceEventId]) @@index([workflowId, contactId]) // Index for querying executions by workflow and contact @@index([workflowId, status]) @@index([contactId, status]) @@index([status, currentStepId]) + @@index([sourceEventId]) @@map("workflow_executions") } @@ -501,6 +510,11 @@ model WorkflowStepExecution { step WorkflowStep @relation(fields: [stepId], references: [id], onDelete: Cascade) stepId String + // Event that claimed this wait for resumption. It keeps a failed continuation + // discoverable by the same durable event while excluding later events/timeouts. + resumeEvent Event? @relation(fields: [resumeEventId], references: [id], onDelete: SetNull) + resumeEventId String? + // Execution state status StepExecutionStatus @default(PENDING) @@ -524,6 +538,7 @@ model WorkflowStepExecution { @@index([executionId, status]) @@index([stepId]) + @@index([resumeEventId]) @@index([status, scheduledFor]) // For delay queue processor @@index([scheduledFor]) @@map("workflow_step_executions") @@ -634,6 +649,19 @@ model Event { email Email? @relation(fields: [emailId], references: [id], onDelete: Cascade) emailId String? + workflowExecutions WorkflowExecution[] + resumedStepExecutions WorkflowStepExecution[] + + // Workflow dispatch is separate from durable event ingestion. Null means a + // committed event still needs dispatch or reconciliation. + processedAt DateTime? + dispatchAttempts Int @default(0) + nextDispatchAt DateTime? + dispatchFailedAt DateTime? + dispatchError String? + dispatchLeaseId String? + dispatchLeaseExpiresAt DateTime? + // Timestamps createdAt DateTime @default(now()) @@ -641,6 +669,9 @@ model Event { @@index([contactId]) @@index([emailId]) @@index([createdAt]) + @@index([processedAt, createdAt]) + // The partial reconciliation index lives in its SQL migration because Prisma + // schema syntax cannot express its pending-only predicate. @@index([projectId, contactId, name, createdAt]) // For event-based segment queries (fast!) @@index([contactId, name, createdAt]) // For per-contact event lookups @@index([projectId, name, createdAt]) // For event stats and analytics by type over time @@ -679,6 +710,31 @@ model IdempotencyKey { @@map("idempotency_keys") } +// ============================================ +// SNS WEBHOOK RECEIPTS +// ============================================ + +model SnsWebhookReceipt { + id String @id @default(uuid()) + + // AWS signs the outer SNS MessageId, making it the durable delivery identity. + messageId String @unique + + // A token-scoped claim prevents concurrent deliveries from processing together. + // Failed claims can be retried immediately; abandoned claims become reclaimable. + status SnsWebhookReceiptStatus @default(PROCESSING) + processingToken String + processingStartedAt DateTime @default(now()) + completedAt DateTime? + expiresAt DateTime + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([expiresAt]) + @@map("sns_webhook_receipts") +} + // ============================================ // ENUMS // ============================================ @@ -691,6 +747,12 @@ enum ProjectDisabledReason { MANUAL // Disabled by support/admin (e.g. directly in DB) } +enum SnsWebhookReceiptStatus { + PROCESSING + COMPLETED + FAILED +} + enum CardVerificationStatus { PENDING // Off-session charge queued or in flight VERIFIED // Card accepted a merchant-initiated charge @@ -754,6 +816,11 @@ enum WorkflowStepType { UPDATE_CONTACT // Update contact fields } +enum WorkflowWaitOutcome { + EVENT + TIMEOUT +} + enum WorkflowExecutionStatus { RUNNING // Currently executing WAITING // Waiting for event or delay diff --git a/packages/shared/src/schemas/index.ts b/packages/shared/src/schemas/index.ts index 0087d8ffa..e3648daa4 100644 --- a/packages/shared/src/schemas/index.ts +++ b/packages/shared/src/schemas/index.ts @@ -1,4 +1,4 @@ -import {CampaignAudienceType, TemplateType, TrackingMode, WorkflowStepType, WorkflowTriggerType} from '@plunk/db'; +import {CampaignAudienceType, TemplateType, TrackingMode, WorkflowStepType, WorkflowTriggerType, WorkflowWaitOutcome} from '@plunk/db'; import type {FilterCondition, FilterGroup} from '@plunk/types'; import {z} from 'zod'; @@ -263,6 +263,7 @@ export const WorkflowSchemas = { fromStepId: uuid, toStepId: uuid, condition: jsonSchema.optional(), + waitOutcome: z.nativeEnum(WorkflowWaitOutcome).optional(), priority: z.number().int().min(0).default(0), }), startExecution: z.object({ diff --git a/packages/types/src/prisma/extended.ts b/packages/types/src/prisma/extended.ts index 40e0cd65c..523176bec 100644 --- a/packages/types/src/prisma/extended.ts +++ b/packages/types/src/prisma/extended.ts @@ -64,6 +64,7 @@ export type WorkflowStepWithTransitions = WorkflowStep & { outgoingTransitions?: Array<{ id: string; condition: Prisma.JsonValue; + waitOutcome: WorkflowTransition['waitOutcome']; priority: number; toStep: WorkflowStep; }>; diff --git a/test/setup.ts b/test/setup.ts index 8cf6c0913..2cc5f676b 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -38,13 +38,14 @@ process.env.NODE_ENV = 'test'; // with, since those two ship empty. Tests never reach SES, so fill in placeholders // rather than requiring every contributor to invent credentials. // -// Only these two: every other required var (JWT_SECRET, the *_URI values, -// AWS_SES_REGION, DATABASE_URL, REDIS_URL) ships with a value in .env.example and -// is set by the CI workflow, and the DB/Redis URLs must point at real services. +// Only supply safe test placeholders for required values that may be absent. +// Database and Redis URLs must still point at real isolated test services. const TEST_ENV_DEFAULTS: Record = { JWT_SECRET: 'test-jwt-secret-key-for-testing', AWS_SES_ACCESS_KEY_ID: 'test-ses-access-key-id', AWS_SES_SECRET_ACCESS_KEY: 'test-ses-secret-access-key', + SNS_TOPIC_ARNS: + 'arn:aws:sns:us-east-1:123456789012:plunk-ses-events,arn:aws:sns:eu-west-1:123456789012:plunk-ses-inbound', }; for (const [key, value] of Object.entries(TEST_ENV_DEFAULTS)) { diff --git a/turbo.json b/turbo.json index c99fb6d54..91510ec64 100644 --- a/turbo.json +++ b/turbo.json @@ -52,6 +52,7 @@ "AWS_SES_REGION", "AWS_SES_ACCESS_KEY_ID", "AWS_SES_SECRET_ACCESS_KEY", + "SNS_TOPIC_ARNS", "SES_CONFIGURATION_SET", "SES_CONFIGURATION_SET_NO_TRACKING", "EMAIL_RATE_LIMIT_PER_SECOND", @@ -118,6 +119,7 @@ "AWS_SES_REGION", "AWS_SES_ACCESS_KEY_ID", "AWS_SES_SECRET_ACCESS_KEY", + "SNS_TOPIC_ARNS", "SES_CONFIGURATION_SET", "SES_CONFIGURATION_SET_NO_TRACKING", "EMAIL_RATE_LIMIT_PER_SECOND", @@ -175,6 +177,7 @@ "AWS_SES_REGION", "AWS_SES_ACCESS_KEY_ID", "AWS_SES_SECRET_ACCESS_KEY", + "SNS_TOPIC_ARNS", "SES_CONFIGURATION_SET", "SES_CONFIGURATION_SET_NO_TRACKING", "EMAIL_RATE_LIMIT_PER_SECOND",