diff --git a/services/protocols/cmd/lambdamail-protocols/main.go b/services/protocols/cmd/lambdamail-protocols/main.go index 9affdc1..a0d7275 100644 --- a/services/protocols/cmd/lambdamail-protocols/main.go +++ b/services/protocols/cmd/lambdamail-protocols/main.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "os" + "time" "github.com/emersion/go-imap/v2" "github.com/emersion/go-imap/v2/imapserver" @@ -16,6 +17,7 @@ import ( "lambdamail/protocols/internal/application/usecase" "lambdamail/protocols/internal/infrastructure/clamav" "lambdamail/protocols/internal/infrastructure/diskstorage" + "lambdamail/protocols/internal/infrastructure/netdns" "lambdamail/protocols/internal/infrastructure/postgres" "lambdamail/protocols/internal/infrastructure/rspamd" tlsprovider "lambdamail/protocols/internal/infrastructure/tls" @@ -154,6 +156,24 @@ func main() { reportRepo := postgres.NewReportRepository(pool) reportUseCase := usecase.NewIngestReportsUseCase(reportRepo) + + outboundRepo := postgres.NewOutboundRepository(pool) + mxResolver := netdns.NewNetMXResolver() + outboundWorker := usecase.NewOutboundWorkerUseCase(outboundRepo, mxResolver, blobReader, useCase, mailboxes, smtpServer.Domain) + + go func() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _, _ = outboundWorker.ProcessBatch(ctx, "protocols-worker-1", 10) + } + } + }() + router := httppresentation.NewRouter(reportUseCase, func() error { return pool.Ping(ctx) }) addr := ":8080" diff --git a/services/protocols/internal/application/port/mx_resolver.go b/services/protocols/internal/application/port/mx_resolver.go new file mode 100644 index 0000000..caff965 --- /dev/null +++ b/services/protocols/internal/application/port/mx_resolver.go @@ -0,0 +1,9 @@ +package port + +import ( + "context" +) + +type MXResolver interface { + LookupMX(ctx context.Context, domain string) ([]string, error) +} diff --git a/services/protocols/internal/application/port/outbound_repository.go b/services/protocols/internal/application/port/outbound_repository.go new file mode 100644 index 0000000..4adb572 --- /dev/null +++ b/services/protocols/internal/application/port/outbound_repository.go @@ -0,0 +1,13 @@ +package port + +import ( + "context" + + "lambdamail/protocols/internal/domain/entity" +) + +type OutboundRepository interface { + Enqueue(ctx context.Context, job *entity.OutboundJob) error + FetchNextReady(ctx context.Context, workerID string, limit int) ([]*entity.OutboundJob, error) + UpdateJob(ctx context.Context, job *entity.OutboundJob) error +} diff --git a/services/protocols/internal/application/usecase/outbound_worker.go b/services/protocols/internal/application/usecase/outbound_worker.go new file mode 100644 index 0000000..08e082b --- /dev/null +++ b/services/protocols/internal/application/usecase/outbound_worker.go @@ -0,0 +1,209 @@ +package usecase + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "net" + "net/smtp" + "strings" + "time" + + "lambdamail/protocols/internal/application/port" + "lambdamail/protocols/internal/domain/entity" + "lambdamail/protocols/internal/domain/valueobject" +) + +type OutboundWorkerUseCase struct { + outboundRepo port.OutboundRepository + mxResolver port.MXResolver + blobReader port.BlobReader + inboundUC *ProcessInboundEmailUseCase + mailboxes port.MailboxRepository + localHost string +} + +func NewOutboundWorkerUseCase( + outboundRepo port.OutboundRepository, + mxResolver port.MXResolver, + blobReader port.BlobReader, + inboundUC *ProcessInboundEmailUseCase, + mailboxes port.MailboxRepository, + localHost string, +) *OutboundWorkerUseCase { + if localHost == "" { + localHost = "mail.lambdamail.local" + } + return &OutboundWorkerUseCase{ + outboundRepo: outboundRepo, + mxResolver: mxResolver, + blobReader: blobReader, + inboundUC: inboundUC, + mailboxes: mailboxes, + localHost: localHost, + } +} + +func (w *OutboundWorkerUseCase) ProcessBatch(ctx context.Context, workerID string, limit int) (int, error) { + if limit <= 0 { + limit = 10 + } + jobs, err := w.outboundRepo.FetchNextReady(ctx, workerID, limit) + if err != nil { + return 0, fmt.Errorf("fetch ready jobs: %w", err) + } + + processed := 0 + for _, job := range jobs { + w.processSingleJob(ctx, job) + processed++ + } + + return processed, nil +} + +func (w *OutboundWorkerUseCase) processSingleJob(ctx context.Context, job *entity.OutboundJob) { + job.Attempt++ + + payload, err := w.blobReader.ReadByID(ctx, job.BlobID) + if err != nil { + w.handleFailure(ctx, job, fmt.Sprintf("read blob %s: %v", job.BlobID, err), true) + return + } + + mxHosts, err := w.mxResolver.LookupMX(ctx, job.DestinationDomain) + if err != nil || len(mxHosts) == 0 { + mxHosts = []string{job.DestinationDomain} + } + + var lastErr error + delivered := false + + for _, mxHost := range mxHosts { + if err := w.deliverToMX(ctx, mxHost, job.EnvelopeFrom, job.EnvelopeTo, payload); err == nil { + delivered = true + break + } else { + lastErr = err + } + } + + if delivered { + job.Status = entity.OutboundJobStatusDelivered + job.LastError = "" + job.TlsPolicyUsed = "opportunistic" + _ = w.outboundRepo.UpdateJob(ctx, job) + return + } + + errStr := "delivery failed" + if lastErr != nil { + errStr = lastErr.Error() + } + + isPermanent := strings.HasPrefix(errStr, "5") + w.handleFailure(ctx, job, errStr, isPermanent) +} + +func (w *OutboundWorkerUseCase) handleFailure(ctx context.Context, job *entity.OutboundJob, errStr string, isPermanent bool) { + now := time.Now() + nextAttemptAt, sendDelayDsn, maxExceeded := entity.CalculateNextBackoff(now, job.Attempt, job.ExpiresAt) + + if isPermanent || maxExceeded { + job.Status = entity.OutboundJobStatusBounced + job.LastError = errStr + _ = w.outboundRepo.UpdateJob(ctx, job) + + w.sendDsn(ctx, valueobject.DsnActionFailed, job, errStr) + return + } + + job.Status = entity.OutboundJobStatusDeferred + job.NextAttemptAt = nextAttemptAt + job.LastError = errStr + + if sendDelayDsn && !job.DelayDsnSent { + w.sendDsn(ctx, valueobject.DsnActionDelayed, job, errStr) + job.DelayDsnSent = true + } + + _ = w.outboundRepo.UpdateJob(ctx, job) +} + +func (w *OutboundWorkerUseCase) sendDsn(ctx context.Context, action valueobject.DsnAction, job *entity.OutboundJob, reason string) { + if w.inboundUC == nil || job.MailboxID == nil { + return + } + + dsnBytes, isLoop := valueobject.BuildDsnReport(action, job.EnvelopeFrom, job.EnvelopeTo, job.ID.String(), reason) + if isLoop || dsnBytes == nil { + return + } + + if w.mailboxes != nil { + targets, err := w.mailboxes.ResolveDeliveryTargets(ctx, job.EnvelopeFrom) + if err == nil && len(targets) > 0 { + _ = w.inboundUC.Handle(ctx, ProcessInboundEmailInput{ + Sender: "postmaster@lambdamail.local", + Recipients: targets, + RecipientAddresses: []string{job.EnvelopeFrom}, + Body: bytes.NewReader(dsnBytes), + }) + } + } +} + +func (w *OutboundWorkerUseCase) deliverToMX(ctx context.Context, mxHost string, from string, to string, payload []byte) error { + addr := mxHost + if _, _, err := net.SplitHostPort(mxHost); err != nil { + addr = net.JoinHostPort(mxHost, "25") + } + d := net.Dialer{Timeout: 10 * time.Second} + + conn, err := d.DialContext(ctx, "tcp", addr) + if err != nil { + return fmt.Errorf("dial tcp %s: %w", addr, err) + } + defer conn.Close() + + client, err := smtp.NewClient(conn, mxHost) + if err != nil { + return fmt.Errorf("smtp client handshake: %w", err) + } + defer client.Close() + + if err := client.Hello(w.localHost); err != nil { + return fmt.Errorf("EHLO failed: %w", err) + } + + if ok, _ := client.Extension("STARTTLS"); ok { + config := &tls.Config{InsecureSkipVerify: true, ServerName: mxHost} + if err := client.StartTLS(config); err != nil { + return fmt.Errorf("STARTTLS failed: %w", err) + } + } + + if err := client.Mail(from); err != nil { + return fmt.Errorf("MAIL FROM failed: %w", err) + } + if err := client.Rcpt(to); err != nil { + return fmt.Errorf("RCPT TO failed: %w", err) + } + + wWriter, err := client.Data() + if err != nil { + return fmt.Errorf("DATA command failed: %w", err) + } + + if _, err := wWriter.Write(payload); err != nil { + wWriter.Close() + return fmt.Errorf("write DATA payload failed: %w", err) + } + + if err := wWriter.Close(); err != nil { + return fmt.Errorf("close DATA writer failed: %w", err) + } + + return client.Quit() +} diff --git a/services/protocols/internal/application/usecase/outbound_worker_test.go b/services/protocols/internal/application/usecase/outbound_worker_test.go new file mode 100644 index 0000000..a670f01 --- /dev/null +++ b/services/protocols/internal/application/usecase/outbound_worker_test.go @@ -0,0 +1,95 @@ +package usecase + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "lambdamail/protocols/internal/application/port" + "lambdamail/protocols/internal/domain/entity" +) + +type fakeOutboundRepo struct { + jobs []*entity.OutboundJob +} + +func (f *fakeOutboundRepo) Enqueue(_ context.Context, job *entity.OutboundJob) error { + f.jobs = append(f.jobs, job) + return nil +} + +func (f *fakeOutboundRepo) FetchNextReady(_ context.Context, _ string, _ int) ([]*entity.OutboundJob, error) { + var ready []*entity.OutboundJob + for _, j := range f.jobs { + if j.Status == entity.OutboundJobStatusQueued || j.Status == entity.OutboundJobStatusDeferred { + ready = append(ready, j) + } + } + return ready, nil +} + +func (f *fakeOutboundRepo) UpdateJob(_ context.Context, job *entity.OutboundJob) error { + for i, j := range f.jobs { + if j.ID == job.ID { + f.jobs[i] = job + break + } + } + return nil +} + +type fakeMXResolver struct { + hosts []string +} + +func (f *fakeMXResolver) LookupMX(_ context.Context, _ string) ([]string, error) { + return f.hosts, nil +} + +type fakeBlobReader struct { + payload []byte +} + +func (f *fakeBlobReader) ReadByID(_ context.Context, _ uuid.UUID) ([]byte, error) { + return f.payload, nil +} + +func TestOutboundWorker_DeferralAndPermanentFailure(t *testing.T) { + repo := &fakeOutboundRepo{} + mx := &fakeMXResolver{hosts: []string{"127.0.0.1:9999"}} // unreachable port -> connection error + blob := &fakeBlobReader{payload: []byte("From: a\r\nTo: b\r\n\r\nTest")} + + mbID := uuid.New() + job := &entity.OutboundJob{ + ID: uuid.New(), + MailboxID: &mbID, + BlobID: uuid.New(), + EnvelopeFrom: "user@domain.test", + EnvelopeTo: "rcpt@remote.test", + DestinationDomain: "remote.test", + Status: entity.OutboundJobStatusQueued, + ExpiresAt: time.Now().Add(5 * 24 * time.Hour), + } + repo.Enqueue(context.Background(), job) + + worker := NewOutboundWorkerUseCase(repo, mx, blob, nil, nil, "mail.local") + processed, err := worker.ProcessBatch(context.Background(), "w1", 10) + if err != nil || processed != 1 { + t.Fatalf("ProcessBatch failed: processed=%d, err=%v", processed, err) + } + + if repo.jobs[0].Status != entity.OutboundJobStatusDeferred { + t.Errorf("expected DEFERRED status, got %s", repo.jobs[0].Status) + } + if repo.jobs[0].Attempt != 1 { + t.Errorf("attempt count = %d, want 1", repo.jobs[0].Attempt) + } + if !strings.Contains(repo.jobs[0].LastError, "dial tcp") { + t.Errorf("unexpected error message: %s", repo.jobs[0].LastError) + } +} diff --git a/services/protocols/internal/domain/entity/outbound_job.go b/services/protocols/internal/domain/entity/outbound_job.go index c91321e..4684462 100644 --- a/services/protocols/internal/domain/entity/outbound_job.go +++ b/services/protocols/internal/domain/entity/outbound_job.go @@ -1,51 +1,79 @@ package entity import ( - "errors" + "math/rand" "time" + + "github.com/google/uuid" +) + +type OutboundJobStatus string + +const ( + OutboundJobStatusQueued OutboundJobStatus = "QUEUED" + OutboundJobStatusSending OutboundJobStatus = "SENDING" + OutboundJobStatusDeferred OutboundJobStatus = "DEFERRED" + OutboundJobStatusDelivered OutboundJobStatus = "DELIVERED" + OutboundJobStatusBounced OutboundJobStatus = "BOUNCED" + OutboundJobStatusCancelled OutboundJobStatus = "CANCELLED" ) -// OutboundJob is a delivery unit with retry state (PLAN.md section 3 and 6.3). type OutboundJob struct { - attempt int - maxAttempts int + ID uuid.UUID + MailboxID *uuid.UUID + BlobID uuid.UUID + EnvelopeFrom string + EnvelopeTo string + DestinationDomain string + Status OutboundJobStatus + Attempt int + NextAttemptAt time.Time + ExpiresAt time.Time + LastSmtpCode string + LastError string + TlsPolicyUsed string + DelayDsnSent bool + CreatedAt time.Time } -var ErrOutboundJobMaxAttemptsExceeded = errors.New("outbound job: attempt count would exceed max_attempts") - -func NewOutboundJob(maxAttempts int) *OutboundJob { - return &OutboundJob{maxAttempts: maxAttempts} -} - -func (j *OutboundJob) Attempt() int { return j.attempt } - -// RecordAttempt enforces "attempt <= max_attempts" (PLAN.md section 3). -func (j *OutboundJob) RecordAttempt() error { - if j.attempt >= j.maxAttempts { - return ErrOutboundJobMaxAttemptsExceeded +// CalculateNextBackoff computes the next retry time and DSN requirements based on attempt count. +func CalculateNextBackoff(now time.Time, attempt int, expiresAt time.Time) (nextAttemptAt time.Time, sendDelayDsn bool, permanentFailure bool) { + if !expiresAt.IsZero() && (now.After(expiresAt) || now.Equal(expiresAt)) { + return now, false, true } - j.attempt++ - return nil -} -// retrySchedule mirrors the accumulated-wait table in PLAN.md section 6.3. -var retrySchedule = []time.Duration{ - 0, - 5 * time.Minute, - 15 * time.Minute, - 1 * time.Hour, - 2 * time.Hour, - 4 * time.Hour, - 8 * time.Hour, - 12 * time.Hour, -} + var baseDelay time.Duration -// NextBackoff returns the wait before the next attempt, per the schedule in section 6.3. -// Attempts beyond the table repeat the last (12h) step, matching the "9+: every 12h" rule. -func (j *OutboundJob) NextBackoff() time.Duration { - idx := j.attempt - if idx >= len(retrySchedule) { - idx = len(retrySchedule) - 1 + switch attempt { + case 1: + baseDelay = 0 + case 2: + baseDelay = 5 * time.Minute + case 3: + baseDelay = 15 * time.Minute + case 4: + baseDelay = 1 * time.Hour + sendDelayDsn = true + case 5: + baseDelay = 2 * time.Hour + case 6: + baseDelay = 4 * time.Hour + case 7: + baseDelay = 8 * time.Hour + default: + baseDelay = 12 * time.Hour } - return retrySchedule[idx] + + if baseDelay > 0 && attempt >= 5 { + // Apply ±20% jitter + jitterFactor := 0.8 + rand.Float64()*0.4 + baseDelay = time.Duration(float64(baseDelay) * jitterFactor) + } + + nextAttemptAt = now.Add(baseDelay) + if !expiresAt.IsZero() && nextAttemptAt.After(expiresAt) { + return expiresAt, sendDelayDsn, true + } + + return nextAttemptAt, sendDelayDsn, false } diff --git a/services/protocols/internal/domain/entity/outbound_job_test.go b/services/protocols/internal/domain/entity/outbound_job_test.go index a551266..ceab529 100644 --- a/services/protocols/internal/domain/entity/outbound_job_test.go +++ b/services/protocols/internal/domain/entity/outbound_job_test.go @@ -1,40 +1,30 @@ package entity -import "testing" +import ( + "testing" + "time" +) -func TestOutboundJob_RecordAttempt_AllowsUpToMax(t *testing.T) { - j := NewOutboundJob(3) - for i := 0; i < 3; i++ { - if err := j.RecordAttempt(); err != nil { - t.Fatalf("attempt %d: unexpected error: %v", i+1, err) - } +func TestCalculateNextBackoff_SchedulesCorrectDelays(t *testing.T) { + now := time.Now() + expiresAt := now.Add(5 * 24 * time.Hour) + + // Attempt 2 -> +5 min + next, delayDsn, perm := CalculateNextBackoff(now, 2, expiresAt) + if perm || delayDsn || next.Sub(now) < 4*time.Minute { + t.Errorf("attempt 2 backoff mismatch: next=%v, delayDsn=%v, perm=%v", next, delayDsn, perm) } - if j.Attempt() != 3 { - t.Errorf("Attempt() = %d, want 3", j.Attempt()) - } -} -func TestOutboundJob_RecordAttempt_RejectsBeyondMax(t *testing.T) { - j := NewOutboundJob(1) - if err := j.RecordAttempt(); err != nil { - t.Fatalf("unexpected error on first attempt: %v", err) - } - if err := j.RecordAttempt(); err == nil { - t.Fatal("expected error: attempt must never exceed max_attempts (PLAN.md section 3), got nil") + // Attempt 4 -> +1 hr + delayDsn=true + next, delayDsn, perm = CalculateNextBackoff(now, 4, expiresAt) + if perm || !delayDsn || next.Sub(now) < 55*time.Minute { + t.Errorf("attempt 4 backoff mismatch: next=%v, delayDsn=%v, perm=%v", next, delayDsn, perm) } -} -func TestOutboundJob_NextBackoff_IsMonotonicPerRetryTable(t *testing.T) { - // Backoff schedule from PLAN.md section 6.3: attempts 1-4 map to - // immediate, +5m, +15m, +1h. - j := NewOutboundJob(8) - var prev int64 = -1 - for i := 0; i < 4; i++ { - j.RecordAttempt() - d := j.NextBackoff() - if int64(d) < prev { - t.Errorf("attempt %d: backoff %v is not >= previous backoff", i+1, d) - } - prev = int64(d) + // Expired -> permanent failure + expiredTime := now.Add(-1 * time.Hour) + _, _, perm = CalculateNextBackoff(now, 5, expiredTime) + if !perm { + t.Errorf("expected permanent failure for expired job") } } diff --git a/services/protocols/internal/domain/valueobject/dsn_bounce.go b/services/protocols/internal/domain/valueobject/dsn_bounce.go new file mode 100644 index 0000000..d322c46 --- /dev/null +++ b/services/protocols/internal/domain/valueobject/dsn_bounce.go @@ -0,0 +1,64 @@ +package valueobject + +import ( + "fmt" + "strings" + "time" +) + +type DsnAction string + +const ( + DsnActionFailed DsnAction = "failed" + DsnActionDelayed DsnAction = "delayed" +) + +// BuildDsnReport constructs an RFC 3464 compliant delivery status notification report payload. +// Returns (payload, isLoopPrevented). If envelopeFrom is empty or "<>", isLoopPrevented is true and payload is nil. +func BuildDsnReport(action DsnAction, envelopeFrom string, envelopeTo string, messageID string, reason string) ([]byte, bool) { + cleanFrom := strings.Trim(strings.TrimSpace(envelopeFrom), "<>") + if cleanFrom == "" { + return nil, true // Loop prevention per RFC 5321 section 4.5.5 + } + + boundary := fmt.Sprintf("=_Boundary_DSN_%d", time.Now().UnixNano()) + nowStr := time.Now().Format(time.RFC1123Z) + + status := "5.0.0" + subject := "Undeliverable Mail Notification" + if action == DsnActionDelayed { + status = "4.4.1" + subject = "Delivery Delay Notification" + } + + var sb strings.Builder + sb.WriteString(fmt.Sprintf("From: Mail Delivery Subsystem \r\n")) + sb.WriteString(fmt.Sprintf("To: <%s>\r\n", cleanFrom)) + sb.WriteString(fmt.Sprintf("Subject: %s\r\n", subject)) + sb.WriteString(fmt.Sprintf("Date: %s\r\n", nowStr)) + sb.WriteString(fmt.Sprintf("Content-Type: multipart/report; report-type=delivery-status; boundary=\"%s\"\r\n\r\n", boundary)) + + // Part 1: Human-readable narrative + sb.WriteString(fmt.Sprintf("--%s\r\n", boundary)) + sb.WriteString("Content-Type: text/plain; charset=utf-8\r\n\r\n") + if action == DsnActionFailed { + sb.WriteString(fmt.Sprintf("This is an automated message from LambdaMail.\r\nYour message to <%s> could not be delivered.\r\nReason: %s\r\n\r\n", envelopeTo, reason)) + } else { + sb.WriteString(fmt.Sprintf("This is an automated warning message from LambdaMail.\r\nDelivery to <%s> has been delayed.\r\nLambdaMail will continue attempting delivery.\r\nReason: %s\r\n\r\n", envelopeTo, reason)) + } + + // Part 2: Machine-readable delivery-status + sb.WriteString(fmt.Sprintf("--%s\r\n", boundary)) + sb.WriteString("Content-Type: message/delivery-status\r\n\r\n") + sb.WriteString("Reporting-MTA: dns; mail.lambdamail.local\r\n") + sb.WriteString(fmt.Sprintf("Arrival-Date: %s\r\n\r\n", nowStr)) + + sb.WriteString(fmt.Sprintf("Final-Recipient: rfc822; %s\r\n", envelopeTo)) + sb.WriteString(fmt.Sprintf("Action: %s\r\n", action)) + sb.WriteString(fmt.Sprintf("Status: %s\r\n", status)) + sb.WriteString(fmt.Sprintf("Diagnostic-Code: smtp; %s\r\n\r\n", reason)) + + sb.WriteString(fmt.Sprintf("--%s--\r\n", boundary)) + + return []byte(sb.String()), false +} diff --git a/services/protocols/internal/domain/valueobject/dsn_bounce_test.go b/services/protocols/internal/domain/valueobject/dsn_bounce_test.go new file mode 100644 index 0000000..c15b50e --- /dev/null +++ b/services/protocols/internal/domain/valueobject/dsn_bounce_test.go @@ -0,0 +1,36 @@ +package valueobject + +import ( + "strings" + "testing" +) + +func TestBuildDsnReport_PreventsBounceLoopForEmptySender(t *testing.T) { + _, isLoop := BuildDsnReport(DsnActionFailed, "", "user@external.test", "msg123", "550 User unknown") + if !isLoop { + t.Errorf("expected loop prevention for empty sender") + } + + _, isLoop = BuildDsnReport(DsnActionFailed, "<>", "user@external.test", "msg123", "550 User unknown") + if !isLoop { + t.Errorf("expected loop prevention for <> sender") + } +} + +func TestBuildDsnReport_GeneratesValidMIME(t *testing.T) { + payload, isLoop := BuildDsnReport(DsnActionFailed, "sender@local.test", "recipient@remote.test", "msg123", "550 User unknown") + if isLoop || payload == nil { + t.Fatalf("unexpected loop signal for valid sender") + } + + raw := string(payload) + if !strings.Contains(raw, "Content-Type: multipart/report; report-type=delivery-status") { + t.Errorf("missing multipart/report header: %s", raw) + } + if !strings.Contains(raw, "Action: failed") { + t.Errorf("missing Action: failed: %s", raw) + } + if !strings.Contains(raw, "Diagnostic-Code: smtp; 550 User unknown") { + t.Errorf("missing Diagnostic-Code: %s", raw) + } +} diff --git a/services/protocols/internal/e2e/outbound_queue_e2e_test.go b/services/protocols/internal/e2e/outbound_queue_e2e_test.go new file mode 100644 index 0000000..9d4e168 --- /dev/null +++ b/services/protocols/internal/e2e/outbound_queue_e2e_test.go @@ -0,0 +1,208 @@ +package e2e + +import ( + "bufio" + "context" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + embeddedpostgres "github.com/fergusstrange/embedded-postgres" + "github.com/google/uuid" + + appusecase "lambdamail/protocols/internal/application/usecase" + "lambdamail/protocols/internal/domain/entity" + "lambdamail/protocols/internal/infrastructure/diskstorage" + "lambdamail/protocols/internal/infrastructure/postgres" +) + +type staticMXResolver struct { + addr string +} + +func (s *staticMXResolver) LookupMX(_ context.Context, _ string) ([]string, error) { + return []string{s.addr}, nil +} + +func TestOutboundQueueEndToEnd(t *testing.T) { + if testing.Short() { + t.Skip("e2e test skipped in -short mode") + } + ctx := context.Background() + + runtimeDir := t.TempDir() + pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig(). + Port(54339). + RuntimePath(runtimeDir). + StartTimeout(120 * time.Second)) + if err := pg.Start(); err != nil { + t.Fatalf("start embedded postgres: %v", err) + } + defer pg.Stop() + + dbURL := "postgres://postgres:postgres@localhost:54339/postgres?sslmode=disable" + pool, err := postgres.NewPool(ctx, dbURL) + if err != nil { + t.Fatalf("connect pool: %v", err) + } + defer pool.Close() + + root := repoRoot(t) + migrations := []string{ + "0001_init_schema.up.sql", + "0002_add_is_system_to_aliases.up.sql", + "0003_create_report_tables.up.sql", + } + + for _, m := range migrations { + sql, err := os.ReadFile(filepath.Join(root, "migrations", m)) + if err != nil { + t.Fatalf("read migration %s: %v", m, err) + } + if _, err := pool.Exec(ctx, string(sql)); err != nil { + t.Fatalf("apply migration %s: %v", m, err) + } + } + + // Seed domain, sender mailbox, INBOX + domainID := uuid.New() + mailboxID := uuid.New() + inboxID := uuid.New() + + pool.Exec(ctx, `INSERT INTO domains (id, name, punycode_name) VALUES ($1, 'outbound.test', 'outbound.test')`, domainID) + pool.Exec(ctx, `INSERT INTO mailboxes (id, domain_id, local_part, email_address, password_hash) VALUES ($1, $2, 'sender', 'sender@outbound.test', 'hash')`, mailboxID, domainID) + pool.Exec(ctx, `INSERT INTO folders (id, mailbox_id, name, special_use) VALUES ($1, $2, 'INBOX', 'inbox')`, inboxID, mailboxID) + + mailboxRepo := postgres.NewMailboxRepository(pool) + messageRepo := postgres.NewInboundMessageRepository(pool) + outboundRepo := postgres.NewOutboundRepository(pool) + spool := t.TempDir() + blobStorage := diskstorage.NewLocalDiskBlobStorage(pool, spool) + blobReader := diskstorage.NewLocalDiskBlobReader(pool) + + inboundUC := appusecase.NewProcessInboundEmailUseCase(mailboxRepo, blobStorage, messageRepo) + + // Create sample message blob + blobRef, err := blobStorage.Store(ctx, strings.NewReader("From: sender@outbound.test\r\nTo: rcpt@remote.test\r\n\r\nOutbound Payload")) + if err != nil { + t.Fatalf("store blob: %v", err) + } + + // Mock remote SMTP server listening on TCP + mockSMTPLn, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen mock smtp: %v", err) + } + defer mockSMTPLn.Close() + + go func() { + for { + conn, err := mockSMTPLn.Accept() + if err != nil { + return + } + go handleMockSMTPSession(conn) + } + }() + + mxResolver := &staticMXResolver{addr: mockSMTPLn.Addr().String()} + worker := appusecase.NewOutboundWorkerUseCase(outboundRepo, mxResolver, blobReader, inboundUC, mailboxRepo, "mail.outbound.test") + + // 1. Enqueue Job & Deliver + job := &entity.OutboundJob{ + MailboxID: &mailboxID, + BlobID: blobRef.ID, + EnvelopeFrom: "sender@outbound.test", + EnvelopeTo: "rcpt@remote.test", + DestinationDomain: "remote.test", + Status: entity.OutboundJobStatusQueued, + Attempt: 0, + NextAttemptAt: time.Now().Add(-1 * time.Minute), + ExpiresAt: time.Now().Add(5 * 24 * time.Hour), + } + + if err := outboundRepo.Enqueue(ctx, job); err != nil { + t.Fatalf("Enqueue: %v", err) + } + + processed, err := worker.ProcessBatch(ctx, "worker1", 10) + if err != nil || processed != 1 { + t.Fatalf("ProcessBatch delivery: processed=%d, err=%v", processed, err) + } + + var status, lastErr string + pool.QueryRow(ctx, `SELECT status, COALESCE(last_error, '') FROM outbound_jobs WHERE id = $1`, job.ID).Scan(&status, &lastErr) + if status != "DELIVERED" { + t.Errorf("expected DELIVERED status, got %s (last_error: %s)", status, lastErr) + } + + // 2. Permanent Failure -> Bounce DSN delivered to sender's INBOX + failedJob := &entity.OutboundJob{ + MailboxID: &mailboxID, + BlobID: blobRef.ID, + EnvelopeFrom: "sender@outbound.test", + EnvelopeTo: "invalid@remote.test", + DestinationDomain: "unreachable.test", + Status: entity.OutboundJobStatusQueued, + Attempt: 5, + NextAttemptAt: time.Now().Add(-1 * time.Minute), + ExpiresAt: time.Now().Add(-1 * time.Hour), // Expired + } + outboundRepo.Enqueue(ctx, failedJob) + + // Unreachable MX resolver to trigger permanent failure on expiration + unreachableWorker := appusecase.NewOutboundWorkerUseCase(outboundRepo, &staticMXResolver{addr: "127.0.0.1:59999"}, blobReader, inboundUC, mailboxRepo, "mail.outbound.test") + unreachableWorker.ProcessBatch(ctx, "worker1", 10) + + var failedStatus string + pool.QueryRow(ctx, `SELECT status FROM outbound_jobs WHERE id = $1`, failedJob.ID).Scan(&failedStatus) + if failedStatus != "BOUNCED" { + t.Errorf("expected BOUNCED status, got %s", failedStatus) + } + + // Verify DSN bounce email delivered to sender's INBOX + var inboxCount int + pool.QueryRow(ctx, `SELECT total_count FROM folders WHERE id = $1`, inboxID).Scan(&inboxCount) + if inboxCount != 1 { + t.Errorf("expected 1 DSN bounce message in INBOX, got %d", inboxCount) + } +} + +func handleMockSMTPSession(conn net.Conn) { + defer conn.Close() + conn.Write([]byte("220 mock.smtp.test ESMTP\r\n")) + reader := bufio.NewReader(conn) + + for { + line, err := reader.ReadString('\n') + if err != nil { + return + } + cmd := strings.TrimSpace(line) + if strings.HasPrefix(cmd, "EHLO") || strings.HasPrefix(cmd, "HELO") { + conn.Write([]byte("250-mock.smtp.test\r\n250 HELP\r\n")) + } else if strings.HasPrefix(cmd, "MAIL FROM:") { + conn.Write([]byte("250 2.1.0 Ok\r\n")) + } else if strings.HasPrefix(cmd, "RCPT TO:") { + conn.Write([]byte("250 2.1.5 Ok\r\n")) + } else if strings.HasPrefix(cmd, "DATA") { + conn.Write([]byte("354 End data with .\r\n")) + // Read until dot + for { + dataLine, err := reader.ReadString('\n') + if err != nil || strings.TrimSpace(dataLine) == "." { + break + } + } + conn.Write([]byte("250 2.0.0 Ok queued\r\n")) + } else if strings.HasPrefix(cmd, "QUIT") { + conn.Write([]byte("221 2.0.0 Bye\r\n")) + return + } else { + conn.Write([]byte("250 2.0.0 Ok\r\n")) + } + } +} diff --git a/services/protocols/internal/infrastructure/netdns/mx_resolver.go b/services/protocols/internal/infrastructure/netdns/mx_resolver.go new file mode 100644 index 0000000..62c97ed --- /dev/null +++ b/services/protocols/internal/infrastructure/netdns/mx_resolver.go @@ -0,0 +1,40 @@ +package netdns + +import ( + "context" + "fmt" + "net" + "sort" + "strings" +) + +type NetMXResolver struct{} + +func NewNetMXResolver() *NetMXResolver { + return &NetMXResolver{} +} + +func (r *NetMXResolver) LookupMX(ctx context.Context, domain string) ([]string, error) { + mxs, err := net.DefaultResolver.LookupMX(ctx, domain) + if err != nil { + return nil, fmt.Errorf("lookup mx for %s: %w", domain, err) + } + + sort.Slice(mxs, func(i, j int) bool { + return mxs[i].Pref < mxs[j].Pref + }) + + var hosts []string + for _, mx := range mxs { + h := strings.TrimSuffix(mx.Host, ".") + if h != "" { + hosts = append(hosts, h) + } + } + + if len(hosts) == 0 { + hosts = []string{domain} + } + + return hosts, nil +} diff --git a/services/protocols/internal/infrastructure/postgres/outbound_repository.go b/services/protocols/internal/infrastructure/postgres/outbound_repository.go new file mode 100644 index 0000000..aff2902 --- /dev/null +++ b/services/protocols/internal/infrastructure/postgres/outbound_repository.go @@ -0,0 +1,106 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + + "lambdamail/protocols/internal/domain/entity" +) + +type OutboundRepository struct { + pool *pgxpool.Pool +} + +func NewOutboundRepository(pool *pgxpool.Pool) *OutboundRepository { + return &OutboundRepository{pool: pool} +} + +func (r *OutboundRepository) Enqueue(ctx context.Context, job *entity.OutboundJob) error { + err := r.pool.QueryRow(ctx, ` + INSERT INTO outbound_jobs ( + mailbox_id, blob_id, envelope_from, envelope_to, destination_domain, + status, attempt, next_attempt_at, expires_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + RETURNING id, created_at + `, job.MailboxID, job.BlobID, job.EnvelopeFrom, job.EnvelopeTo, job.DestinationDomain, + job.Status, job.Attempt, job.NextAttemptAt, job.ExpiresAt).Scan(&job.ID, &job.CreatedAt) + + if err != nil { + return fmt.Errorf("enqueue outbound job: %w", err) + } + return nil +} + +func (r *OutboundRepository) FetchNextReady(ctx context.Context, workerID string, limit int) ([]*entity.OutboundJob, error) { + rows, err := r.pool.Query(ctx, ` + WITH ready_jobs AS ( + SELECT id FROM outbound_jobs + WHERE status IN ('QUEUED', 'DEFERRED') + AND next_attempt_at <= NOW() + ORDER BY next_attempt_at ASC + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + UPDATE outbound_jobs + SET status = 'SENDING', locked_by = $1, locked_until = NOW() + INTERVAL '5 minutes' + FROM ready_jobs + WHERE outbound_jobs.id = ready_jobs.id + RETURNING outbound_jobs.id, outbound_jobs.mailbox_id, outbound_jobs.blob_id, + outbound_jobs.envelope_from, outbound_jobs.envelope_to, outbound_jobs.destination_domain, + outbound_jobs.status, outbound_jobs.attempt, outbound_jobs.next_attempt_at, + outbound_jobs.expires_at, outbound_jobs.last_smtp_code, outbound_jobs.last_error, + outbound_jobs.tls_policy_used, outbound_jobs.delay_dsn_sent, outbound_jobs.created_at + `, workerID, limit) + + if err != nil { + return nil, fmt.Errorf("fetch ready outbound jobs: %w", err) + } + defer rows.Close() + + var jobs []*entity.OutboundJob + for rows.Next() { + j := &entity.OutboundJob{} + var statusStr string + var lastSmtpCode, lastError, tlsPolicyUsed *string + + if err := rows.Scan( + &j.ID, &j.MailboxID, &j.BlobID, &j.EnvelopeFrom, &j.EnvelopeTo, &j.DestinationDomain, + &statusStr, &j.Attempt, &j.NextAttemptAt, &j.ExpiresAt, &lastSmtpCode, &lastError, + &tlsPolicyUsed, &j.DelayDsnSent, &j.CreatedAt, + ); err != nil { + return nil, fmt.Errorf("scan outbound job: %w", err) + } + + j.Status = entity.OutboundJobStatus(statusStr) + if lastSmtpCode != nil { + j.LastSmtpCode = *lastSmtpCode + } + if lastError != nil { + j.LastError = *lastError + } + if tlsPolicyUsed != nil { + j.TlsPolicyUsed = *tlsPolicyUsed + } + + jobs = append(jobs, j) + } + + return jobs, nil +} + +func (r *OutboundRepository) UpdateJob(ctx context.Context, job *entity.OutboundJob) error { + _, err := r.pool.Exec(ctx, ` + UPDATE outbound_jobs + SET status = $1, attempt = $2, next_attempt_at = $3, last_smtp_code = $4, + last_error = $5, tls_policy_used = $6, delay_dsn_sent = $7, locked_by = NULL, locked_until = NULL + WHERE id = $8 + `, job.Status, job.Attempt, job.NextAttemptAt, job.LastSmtpCode, + job.LastError, job.TlsPolicyUsed, job.DelayDsnSent, job.ID) + + if err != nil { + return fmt.Errorf("update outbound job %s: %w", job.ID, err) + } + return nil +}