diff --git a/.gitignore b/.gitignore
index e77d1a9..ddad1f7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,5 +34,15 @@ Thumbs.db
# TypeScript incremental build cache
*.tsbuildinfo
-*json
+
+# DMARC and TLS-RPT reports. They arrive by mail, are parsed into the report
+# tables on delivery, and are named for the reporter and the date range: data,
+# never source.
+#
+# The pattern here was "*json" - no dot - which ignored every .json file in the
+# repository. The ones already committed survived only because .gitignore does
+# not apply to tracked files; any new one - another language's messages, a new
+# service's package.json - would have been silently left out of every commit.
*.json.gz
+*.xml.gz
+*.zip.gz
diff --git a/migrations/0008_add_reports_folder.down.sql b/migrations/0008_add_reports_folder.down.sql
new file mode 100644
index 0000000..09e5bb0
--- /dev/null
+++ b/migrations/0008_add_reports_folder.down.sql
@@ -0,0 +1,8 @@
+-- Only removes the folder where it is still empty. A Reports folder holding
+-- delivered reports is data, and dropping it would cascade those messages away.
+DELETE FROM folders f
+ WHERE LOWER(f.name) = 'reports'
+ AND f.special_use IS NULL
+ AND NOT EXISTS (
+ SELECT 1 FROM email_messages e WHERE e.folder_id = f.id
+ );
diff --git a/migrations/0008_add_reports_folder.up.sql b/migrations/0008_add_reports_folder.up.sql
new file mode 100644
index 0000000..489eec2
--- /dev/null
+++ b/migrations/0008_add_reports_folder.up.sql
@@ -0,0 +1,17 @@
+-- Give every existing mailbox a Reports folder.
+--
+-- DMARC and TLS-RPT reports are delivered to the dmarc@ and tlsrpt@ aliases
+-- and are now parsed on arrival and filed here rather than into the inbox.
+-- Without the folder the delivery path falls back to INBOX, which is the noise
+-- this is meant to remove.
+--
+-- IMAP defines no special-use role for reports, so the folder is matched by
+-- name and special_use stays NULL.
+INSERT INTO folders (mailbox_id, name, special_use)
+SELECT m.id, 'Reports', NULL
+ FROM mailboxes m
+ WHERE NOT EXISTS (
+ SELECT 1 FROM folders f
+ WHERE f.mailbox_id = m.id
+ AND LOWER(f.name) = 'reports'
+ );
diff --git a/migrations/0009_report_ingestion_idempotency.down.sql b/migrations/0009_report_ingestion_idempotency.down.sql
new file mode 100644
index 0000000..5ac578b
--- /dev/null
+++ b/migrations/0009_report_ingestion_idempotency.down.sql
@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS uq_tls_rpt_reports_org_report;
+DROP INDEX IF EXISTS uq_dmarc_reports_org_report;
diff --git a/migrations/0009_report_ingestion_idempotency.up.sql b/migrations/0009_report_ingestion_idempotency.up.sql
new file mode 100644
index 0000000..4d70d8c
--- /dev/null
+++ b/migrations/0009_report_ingestion_idempotency.up.sql
@@ -0,0 +1,35 @@
+-- Make storing a report idempotent, keyed on the reporter and their own id
+-- for the report.
+--
+-- Reports used to be written only by hand through the ingest API, so a
+-- duplicate took deliberate effort. They are now parsed automatically out of
+-- the mail that carries them, which makes redelivery routine: an SMTP retry
+-- after a slow commit, or a reporter that simply sends again, would otherwise
+-- store the report a second time and double every number the admin console
+-- shows.
+--
+-- tls_rpt_reports had no unique key at all. dmarc_reports had one over
+-- (domain_id, org_name, report_id), but the ingest path never fills domain_id
+-- and in Postgres a NULL never equals a NULL - so that key did not prevent a
+-- duplicate either.
+
+-- Collapse the duplicates that already accumulated, keeping the earliest row
+-- of each group so the child records that point at it stay valid.
+DELETE FROM tls_rpt_reports a
+ USING tls_rpt_reports b
+ WHERE a.organization_name = b.organization_name
+ AND a.report_id = b.report_id
+ AND (a.created_at, a.id) > (b.created_at, b.id);
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_tls_rpt_reports_org_report
+ ON tls_rpt_reports (organization_name, report_id);
+
+DELETE FROM dmarc_reports a
+ USING dmarc_reports b
+ WHERE a.org_name = b.org_name
+ AND a.report_id = b.report_id
+ AND a.domain IS NOT DISTINCT FROM b.domain
+ AND a.id > b.id;
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_dmarc_reports_org_report
+ ON dmarc_reports (org_name, report_id);
diff --git a/services/auth/src/repository.ts b/services/auth/src/repository.ts
index 61ef4f6..229370b 100644
--- a/services/auth/src/repository.ts
+++ b/services/auth/src/repository.ts
@@ -686,7 +686,12 @@ export async function createMailbox(scope: AdminScope, input: CreateMailboxInput
`INSERT INTO folders (mailbox_id, name, special_use)
SELECT $1, f.name, f.special_use
FROM (VALUES ('INBOX','inbox'), ('Sent','sent'), ('Drafts','drafts'),
- ('Trash','trash'), ('Junk','junk'), ('Archive','archive')) AS f(name, special_use)`,
+ ('Trash','trash'), ('Junk','junk'), ('Archive','archive'),
+ -- Where delivered DMARC and TLS-RPT reports are filed.
+ -- IMAP has no special-use role for these, so it is matched
+ -- by name. Without the folder the delivery path falls back
+ -- to INBOX, which is the noise this is here to remove.
+ ('Reports', NULL)) AS f(name, special_use)`,
[created!.id],
);
diff --git a/services/protocols/cmd/lambdamail-protocols/backfill_reports.go b/services/protocols/cmd/lambdamail-protocols/backfill_reports.go
new file mode 100644
index 0000000..c293546
--- /dev/null
+++ b/services/protocols/cmd/lambdamail-protocols/backfill_reports.go
@@ -0,0 +1,50 @@
+package main
+
+import (
+ "context"
+ "log"
+ "time"
+
+ "lambdamail/protocols/internal/application/usecase"
+ "lambdamail/protocols/internal/infrastructure/diskstorage"
+ "lambdamail/protocols/internal/infrastructure/postgres"
+)
+
+// runBackfillReports parses the DMARC and TLS-RPT reports that were delivered
+// before automatic ingestion existed.
+//
+// Run by hand, once, after deploying ingestion: every report received until
+// then was filed in the admin's inbox as an attachment nothing ever read, so
+// the tables the console shows start empty however long the server has been
+// running. Safe to run more than once - storing a report is idempotent, and a
+// message already filed in Reports is not picked up again.
+func runBackfillReports(cfg config) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+ defer cancel()
+
+ pool, err := postgres.NewPool(ctx, cfg.DatabaseURL)
+ if err != nil {
+ log.Fatalf("backfill: cannot reach the database: %v", err)
+ }
+ defer pool.Close()
+
+ // The reader alone: the backfill only ever reads stored messages back out.
+ blobs := diskstorage.NewLocalDiskBlobReader(pool)
+
+ store := postgres.NewDeliveredReportStore(pool)
+ uc := usecase.NewBackfillReportsUseCase(store, blobs,
+ usecase.NewIngestDeliveredReportsUseCase(
+ usecase.NewIngestReportsUseCase(postgres.NewReportRepository(pool))))
+
+ summary, err := uc.Run(ctx, 0)
+ if err != nil {
+ log.Fatalf("backfill: %v", err)
+ }
+
+ log.Printf("backfill: scanned %d stored reports, ingested %d, could not read %d",
+ summary.Scanned, summary.Ingested, summary.Failed)
+ if summary.Failed > 0 {
+ log.Printf("backfill: the unreadable ones are filed in Reports with their " +
+ "bytes intact, so this can be run again once the cause is fixed")
+ }
+}
diff --git a/services/protocols/cmd/lambdamail-protocols/main.go b/services/protocols/cmd/lambdamail-protocols/main.go
index 747ad8f..632f8fc 100644
--- a/services/protocols/cmd/lambdamail-protocols/main.go
+++ b/services/protocols/cmd/lambdamail-protocols/main.go
@@ -59,8 +59,11 @@ func main() {
case "healthcheck":
runHealthcheck()
return
+ case "backfill-reports":
+ runBackfillReports(cfg)
+ return
default:
- log.Fatalf("unknown subcommand %q (expected preflight or healthcheck)", os.Args[1])
+ log.Fatalf("unknown subcommand %q (expected preflight, healthcheck or backfill-reports)", os.Args[1])
}
}
@@ -118,6 +121,16 @@ func run(cfg config) {
inboundUC.SetScanner(scanner)
}
+ // DMARC and TLS-RPT reports arrive as ordinary mail to the dmarc@ and
+ // tlsrpt@ aliases every domain gets. Reading them here is what fills the
+ // report tables the admin console shows; without this they arrived as
+ // attachments in the admin's inbox and nothing ever parsed one.
+ //
+ // It shares the report use case with the HTTP ingest endpoints below, so
+ // mail and API writes land in the same tables through the same parsers.
+ reportsUC := usecase.NewIngestReportsUseCase(reportRepo)
+ inboundUC.SetReportIngestor(usecase.NewIngestDeliveredReportsUseCase(reportsUC))
+
// ------------------------------------------------------------ outbound
var signer port.DkimSigner
if dkimRepo != nil {
@@ -173,7 +186,7 @@ func run(cfg config) {
postgres.NewWebmailRepository(pool), blobReader, submissionUC, authRepo, cfg.PrimaryMailHost,
).WithLocalFiling(blobs, messages)
- router := httppresentation.NewRouter(usecase.NewIngestReportsUseCase(reportRepo), func() error { return pool.Ping(ctx) })
+ router := httppresentation.NewRouter(reportsUC, func() error { return pool.Ping(ctx) })
if cfg.JwtSecret == "" {
log.Printf("JWT_SECRET is not set: the webmail mail API stays disabled")
}
diff --git a/services/protocols/internal/application/port/report_repository.go b/services/protocols/internal/application/port/report_repository.go
index 261bd19..cbb7a90 100644
--- a/services/protocols/internal/application/port/report_repository.go
+++ b/services/protocols/internal/application/port/report_repository.go
@@ -3,6 +3,8 @@ package port
import (
"context"
+ "github.com/google/uuid"
+
"lambdamail/protocols/internal/domain/entity"
)
@@ -11,3 +13,21 @@ type ReportRepository interface {
SaveDmarcReport(ctx context.Context, report *entity.DmarcReport) error
SaveTlsRptReport(ctx context.Context, report *entity.TlsRptReport) error
}
+
+// StoredReportMessage is one already-delivered message that looks like a
+// report: it was addressed to a report alias and is still filed in a mailbox.
+type StoredReportMessage struct {
+ MessageID uuid.UUID
+ BlobID uuid.UUID
+ RecipientAddress string
+}
+
+// DeliveredReportStore reaches the reports that were delivered before
+// automatic ingestion existed, so they can be parsed after the fact.
+type DeliveredReportStore interface {
+ // ListUningestedReports returns stored messages addressed to a report
+ // alias. limit caps the batch; 0 means a full pass.
+ ListUningestedReports(ctx context.Context, limit int) ([]StoredReportMessage, error)
+ // MoveToReportsFolder files a backfilled message out of the inbox.
+ MoveToReportsFolder(ctx context.Context, messageID uuid.UUID) error
+}
diff --git a/services/protocols/internal/application/usecase/backfill_reports.go b/services/protocols/internal/application/usecase/backfill_reports.go
new file mode 100644
index 0000000..1e6eb5c
--- /dev/null
+++ b/services/protocols/internal/application/usecase/backfill_reports.go
@@ -0,0 +1,89 @@
+package usecase
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "lambdamail/protocols/internal/application/port"
+)
+
+// BackfillReportsUseCase parses reports that were delivered before automatic
+// ingestion existed.
+//
+// Every report received until now was filed in the admin's inbox as an
+// attachment and never read, so the tables the admin console shows start
+// empty however long the server has been running. The messages are still
+// stored, so the backlog can be parsed after the fact and filed out of the
+// inbox at the same time.
+type BackfillReportsUseCase struct {
+ store port.DeliveredReportStore
+ blobs port.BlobReader
+ ingestor *IngestDeliveredReportsUseCase
+}
+
+func NewBackfillReportsUseCase(
+ store port.DeliveredReportStore,
+ blobs port.BlobReader,
+ ingestor *IngestDeliveredReportsUseCase,
+) *BackfillReportsUseCase {
+ return &BackfillReportsUseCase{store: store, blobs: blobs, ingestor: ingestor}
+}
+
+// BackfillSummary is what one run did, for the operator running it by hand.
+type BackfillSummary struct {
+ // Scanned is how many stored report messages were examined.
+ Scanned int
+ // Ingested is how many produced a report row.
+ Ingested int
+ // Failed is how many could not be read or parsed. They are left filed in
+ // Reports with their bytes intact, so a later run can retry them.
+ Failed int
+}
+
+// Run parses the backlog. limit caps how many messages are examined; 0 means
+// whatever the store considers a full pass.
+//
+// Only a failure to read the backlog itself stops the run. An individual
+// message that cannot be read or parsed is counted and skipped: one corrupt
+// attachment from years ago must not block every report behind it.
+func (uc *BackfillReportsUseCase) Run(ctx context.Context, limit int) (BackfillSummary, error) {
+ summary := BackfillSummary{}
+
+ pending, err := uc.store.ListUningestedReports(ctx, limit)
+ if err != nil {
+ return summary, fmt.Errorf("list the report backlog: %w", err)
+ }
+
+ for _, msg := range pending {
+ summary.Scanned++
+
+ raw, err := uc.blobs.ReadByID(ctx, msg.BlobID)
+ if err != nil {
+ summary.Failed++
+ log.Printf("backfill: could not read the message for %s: %v", msg.RecipientAddress, err)
+ continue
+ }
+
+ outcome, err := uc.ingestor.Ingest(ctx, []string{msg.RecipientAddress}, raw)
+ switch {
+ case err != nil:
+ summary.Failed++
+ log.Printf("backfill: could not ingest the message for %s: %v", msg.RecipientAddress, err)
+ case outcome.Stored:
+ summary.Ingested++
+ default:
+ // Recognised as a report but nothing parseable inside it.
+ summary.Failed++
+ }
+
+ // Filed either way. The inbox is being cleared, and a report this
+ // server cannot parse today is still a report - its bytes stay on
+ // disk, so a later run can try again once the parser is fixed.
+ if err := uc.store.MoveToReportsFolder(ctx, msg.MessageID); err != nil {
+ log.Printf("backfill: could not file %s into %s: %v", msg.MessageID, ReportsFolderName, err)
+ }
+ }
+
+ return summary, nil
+}
diff --git a/services/protocols/internal/application/usecase/backfill_reports_test.go b/services/protocols/internal/application/usecase/backfill_reports_test.go
new file mode 100644
index 0000000..b964d49
--- /dev/null
+++ b/services/protocols/internal/application/usecase/backfill_reports_test.go
@@ -0,0 +1,172 @@
+package usecase
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/google/uuid"
+
+ "lambdamail/protocols/internal/application/port"
+)
+
+// Reports that arrived before ingestion existed are still sitting in the
+// admin's inbox. They are ordinary stored messages, so they can be read back
+// out and parsed after the fact.
+
+type stubDeliveredReportStore struct {
+ pending []port.StoredReportMessage
+ filed []uuid.UUID
+ blobs map[uuid.UUID][]byte
+ listErr error
+ fileErr error
+ blobErr error
+}
+
+func (s *stubDeliveredReportStore) ListUningestedReports(
+ _ context.Context, limit int,
+) ([]port.StoredReportMessage, error) {
+ if s.listErr != nil {
+ return nil, s.listErr
+ }
+ if limit > 0 && len(s.pending) > limit {
+ return s.pending[:limit], nil
+ }
+ return s.pending, nil
+}
+
+func (s *stubDeliveredReportStore) MoveToReportsFolder(_ context.Context, messageID uuid.UUID) error {
+ if s.fileErr != nil {
+ return s.fileErr
+ }
+ s.filed = append(s.filed, messageID)
+ return nil
+}
+
+func (s *stubDeliveredReportStore) ReadByID(_ context.Context, blobID uuid.UUID) ([]byte, error) {
+ if s.blobErr != nil {
+ return nil, s.blobErr
+ }
+ body, ok := s.blobs[blobID]
+ if !ok {
+ return nil, errors.New("blob not found")
+ }
+ return body, nil
+}
+
+func newBackfill(store *stubDeliveredReportStore, repo *recordingReportRepo) *BackfillReportsUseCase {
+ return NewBackfillReportsUseCase(
+ store, store, NewIngestDeliveredReportsUseCase(NewIngestReportsUseCase(repo)))
+}
+
+func TestBackfillIngestsAReportAlreadyInTheMailbox(t *testing.T) {
+ blobID := uuid.New()
+ messageID := uuid.New()
+ payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
+
+ store := &stubDeliveredReportStore{
+ pending: []port.StoredReportMessage{
+ {MessageID: messageID, BlobID: blobID, RecipientAddress: "tlsrpt@example.test"},
+ },
+ blobs: map[uuid.UUID][]byte{blobID: payload},
+ }
+ repo := &recordingReportRepo{}
+
+ summary, err := newBackfill(store, repo).Run(context.Background(), 0)
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+
+ if len(repo.tlsRpt) != 1 {
+ t.Fatalf("the stored report was not ingested: %d", len(repo.tlsRpt))
+ }
+ if repo.tlsRpt[0].Domain != "example.test" {
+ t.Errorf("ingested domain is %q", repo.tlsRpt[0].Domain)
+ }
+ if summary.Ingested != 1 {
+ t.Errorf("summary reports %d ingested, want 1", summary.Ingested)
+ }
+ // Filed out of the inbox, which is the point of running this at all.
+ if len(store.filed) != 1 || store.filed[0] != messageID {
+ t.Errorf("the message was not moved to Reports: %v", store.filed)
+ }
+}
+
+// Running it twice must be safe: the second pass has nothing left to do, and
+// storage is idempotent anyway.
+func TestBackfillOverAnEmptyBacklogDoesNothing(t *testing.T) {
+ store := &stubDeliveredReportStore{}
+ repo := &recordingReportRepo{}
+
+ summary, err := newBackfill(store, repo).Run(context.Background(), 0)
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+ if summary.Ingested != 0 || summary.Failed != 0 {
+ t.Errorf("an empty backlog produced %+v", summary)
+ }
+}
+
+// One unreadable report must not stop the rest of the backlog: a single
+// corrupted attachment from years ago would otherwise block every report
+// behind it forever.
+func TestBackfillContinuesPastAnUnreadableReport(t *testing.T) {
+ goodBlob, badBlob := uuid.New(), uuid.New()
+ goodMsg, badMsg := uuid.New(), uuid.New()
+
+ store := &stubDeliveredReportStore{
+ pending: []port.StoredReportMessage{
+ {MessageID: badMsg, BlobID: badBlob, RecipientAddress: "tlsrpt@example.test"},
+ {MessageID: goodMsg, BlobID: goodBlob, RecipientAddress: "tlsrpt@example.test"},
+ },
+ blobs: map[uuid.UUID][]byte{
+ badBlob: messageWithAttachment("broken.json.gz", "application/gzip", []byte("rubbish")),
+ goodBlob: messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON)),
+ },
+ }
+ repo := &recordingReportRepo{}
+
+ summary, err := newBackfill(store, repo).Run(context.Background(), 0)
+ if err != nil {
+ t.Fatalf("run: %v", err)
+ }
+ if summary.Ingested != 1 {
+ t.Errorf("the readable report was not ingested: %+v", summary)
+ }
+ if summary.Failed != 1 {
+ t.Errorf("the unreadable one was not reported: %+v", summary)
+ }
+ // Both are filed either way - the inbox is being cleared, and a report
+ // this server cannot parse is still a report.
+ if len(store.filed) != 2 {
+ t.Errorf("filed %d messages, want both", len(store.filed))
+ }
+}
+
+// A blob that cannot be read at all is counted and skipped, never fatal.
+func TestBackfillSurvivesAMissingBlob(t *testing.T) {
+ store := &stubDeliveredReportStore{
+ pending: []port.StoredReportMessage{
+ {MessageID: uuid.New(), BlobID: uuid.New(), RecipientAddress: "dmarc@example.test"},
+ },
+ blobs: map[uuid.UUID][]byte{},
+ }
+ repo := &recordingReportRepo{}
+
+ summary, err := newBackfill(store, repo).Run(context.Background(), 0)
+ if err != nil {
+ t.Fatalf("a missing blob must not fail the run: %v", err)
+ }
+ if summary.Failed != 1 {
+ t.Errorf("the missing blob was not counted: %+v", summary)
+ }
+}
+
+// A failure listing the backlog is the one thing worth stopping for: it means
+// the database is unreachable, and every message would be counted as failed.
+func TestBackfillStopsIfItCannotReadTheBacklog(t *testing.T) {
+ store := &stubDeliveredReportStore{listErr: errors.New("database is down")}
+ if _, err := newBackfill(store, &recordingReportRepo{}).Run(context.Background(), 0); err == nil {
+ t.Error("a backlog that cannot be listed should be reported as an error")
+ }
+}
diff --git a/services/protocols/internal/application/usecase/ingest_delivered_reports.go b/services/protocols/internal/application/usecase/ingest_delivered_reports.go
new file mode 100644
index 0000000..0927722
--- /dev/null
+++ b/services/protocols/internal/application/usecase/ingest_delivered_reports.go
@@ -0,0 +1,188 @@
+package usecase
+
+import (
+ "bytes"
+ "context"
+ "io"
+ "log"
+ "strings"
+
+ "github.com/emersion/go-message"
+)
+
+// ReportKind says what a delivered message is, decided by the address it was
+// sent to.
+type ReportKind string
+
+const (
+ // ReportKindNone is ordinary mail, which this use case leaves alone.
+ ReportKindNone ReportKind = ""
+ ReportKindDmarc ReportKind = "dmarc"
+ ReportKindTlsRpt ReportKind = "tlsrpt"
+)
+
+// reportLocalParts maps the mailbox a report is addressed to onto what it
+// contains.
+//
+// The address is the only trustworthy signal. The subject and the attachment
+// filename are both chosen by the sender, so keying on either would let anyone
+// who can send this server a message write rows into the report tables. These
+// local parts exist because EnsureSystemAliases creates them for every domain.
+var reportLocalParts = map[string]ReportKind{
+ "dmarc": ReportKindDmarc,
+ "tlsrpt": ReportKindTlsRpt,
+}
+
+// ReportsFolderName is where an ingested report is filed.
+//
+// Not the inbox: these are machine-to-machine documents that arrive daily from
+// every large receiver, and nobody reads the attachment by hand. Not discarded
+// either - keeping the original bytes is what makes a parser fix re-appliable
+// to reports already received.
+const ReportsFolderName = "Reports"
+
+// IngestDeliveredReportsUseCase reads DMARC and TLS-RPT reports out of the
+// mail that carries them.
+//
+// The parsers, the tables and the HTTP ingest endpoints all existed already,
+// but nothing connected inbound mail to them: reports were filed in the
+// admin's inbox as attachments nobody could read, and the report tables stayed
+// empty however many arrived.
+type IngestDeliveredReportsUseCase struct {
+ reports *IngestReportsUseCase
+}
+
+func NewIngestDeliveredReportsUseCase(reports *IngestReportsUseCase) *IngestDeliveredReportsUseCase {
+ return &IngestDeliveredReportsUseCase{reports: reports}
+}
+
+// Ingest parses any report the message carries and reports what kind of
+// message it was, so the caller can file it somewhere other than the inbox.
+//
+// It never returns an error for a bad report. A malformed payload is the
+// reporter's mistake and a storage failure is this server's, and neither is a
+// reason to answer the SMTP transaction with a failure: the reporter would
+// retry the same message on a schedule, forever, and no retry would ever help.
+// Failures are logged and the message is still delivered, which also leaves
+// the original bytes on disk to re-ingest once the cause is fixed.
+func (uc *IngestDeliveredReportsUseCase) Ingest(
+ ctx context.Context, recipients []string, payload []byte,
+) (ReportOutcome, error) {
+ kind := ReportKindNone
+ for _, address := range recipients {
+ if found := reportKindForAddress(address); found != ReportKindNone {
+ kind = found
+ break
+ }
+ }
+ if kind == ReportKindNone || uc.reports == nil {
+ return ReportOutcome{}, nil
+ }
+
+ // Every part is tried in turn, so the covering note a reporter puts in
+ // front of the attachment is simply one that does not parse. Only the
+ // last failure is kept, and only reported if no part worked at all:
+ // logging each one would file an error for the human-readable part of
+ // every report that ingests perfectly well.
+ var lastErr error
+ for _, candidate := range reportPayloads(payload) {
+ var err error
+ switch kind {
+ case ReportKindDmarc:
+ _, err = uc.reports.IngestDmarc(ctx, candidate)
+ case ReportKindTlsRpt:
+ _, err = uc.reports.IngestTlsRpt(ctx, candidate)
+ }
+ if err != nil {
+ lastErr = err
+ continue
+ }
+ // One report per message, so there is nothing to gain from reading
+ // the remaining parts once one has been stored.
+ return ReportOutcome{Kind: kind, Stored: true}, nil
+ }
+
+ if lastErr != nil {
+ // Logged rather than returned: see the note above on why a bad report
+ // must not fail the delivery.
+ log.Printf("reports: no readable %s report in a message to %v: %v", kind, recipients, lastErr)
+ }
+ // Recognised by its address but nothing could be parsed out of it. Still a
+ // report, so the caller files it with the others rather than leaving it in
+ // the inbox.
+ return ReportOutcome{Kind: kind}, nil
+}
+
+// ReportOutcome says what a message turned out to be and whether anything was
+// actually stored from it.
+//
+// The two are separate because they drive different decisions: Kind decides
+// where the message is filed, and Stored is how a caller tells a report it
+// parsed from one it merely recognised - which Ingest deliberately does not
+// report as an error.
+type ReportOutcome struct {
+ Kind ReportKind
+ Stored bool
+}
+
+// reportKindForAddress classifies one recipient address by its local part.
+func reportKindForAddress(address string) ReportKind {
+ local, _, found := strings.Cut(address, "@")
+ if !found {
+ return ReportKindNone
+ }
+ return reportLocalParts[strings.ToLower(strings.TrimSpace(local))]
+}
+
+// reportPayloads returns every part of the message that could be a report,
+// decoded.
+//
+// Every attachment is offered rather than the one whose name looks right:
+// filenames vary between reporters and the parsers already reject anything
+// that is not a report, so matching on the name would only add a way to miss
+// one. The whole body is offered last, for the reporters that send the
+// document as the message itself rather than as an attachment.
+func reportPayloads(payload []byte) [][]byte {
+ entity, err := message.Read(bytes.NewReader(payload))
+ if err != nil {
+ return [][]byte{payload}
+ }
+
+ candidates := collectReportParts(entity, 0)
+ return append(candidates, payload)
+}
+
+// maxReportParts bounds the walk: a report has one attachment, and a message
+// with hundreds of nested parts is not one this server needs to read.
+const maxReportParts = 16
+
+// maxReportPartBytes bounds each part. Aggregate reports are XML or JSON in
+// the tens of kilobytes even before compression.
+const maxReportPartBytes = 32 << 20
+
+func collectReportParts(entity *message.Entity, depth int) [][]byte {
+ // Bounded so a deliberately deep MIME tree cannot drive this into a
+ // pathological walk.
+ if depth > 8 {
+ return nil
+ }
+ if multipart := entity.MultipartReader(); multipart != nil {
+ out := [][]byte{}
+ for len(out) < maxReportParts {
+ part, err := multipart.NextPart()
+ if err != nil {
+ break
+ }
+ out = append(out, collectReportParts(part, depth+1)...)
+ }
+ return out
+ }
+
+ // go-message decodes the transfer encoding as the body is read, so what
+ // comes back here is the attachment's real bytes rather than its base64.
+ body, err := io.ReadAll(io.LimitReader(entity.Body, maxReportPartBytes))
+ if err != nil || len(body) == 0 {
+ return nil
+ }
+ return [][]byte{body}
+}
diff --git a/services/protocols/internal/application/usecase/ingest_delivered_reports_test.go b/services/protocols/internal/application/usecase/ingest_delivered_reports_test.go
new file mode 100644
index 0000000..ae9a873
--- /dev/null
+++ b/services/protocols/internal/application/usecase/ingest_delivered_reports_test.go
@@ -0,0 +1,318 @@
+package usecase
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "encoding/base64"
+ "errors"
+ "testing"
+
+ "github.com/google/uuid"
+
+ "lambdamail/protocols/internal/application/port"
+ "lambdamail/protocols/internal/domain/entity"
+)
+
+// Reports arrive as ordinary mail to the addresses the system aliases
+// guarantee exist. Nothing read them: they were filed in the admin's inbox as
+// attachments and the parsed tables stayed empty.
+
+type recordingReportRepo struct {
+ dmarc []*entity.DmarcReport
+ tlsRpt []*entity.TlsRptReport
+ err error
+}
+
+func (r *recordingReportRepo) SaveDmarcReport(_ context.Context, report *entity.DmarcReport) error {
+ if r.err != nil {
+ return r.err
+ }
+ r.dmarc = append(r.dmarc, report)
+ return nil
+}
+
+func (r *recordingReportRepo) SaveTlsRptReport(_ context.Context, report *entity.TlsRptReport) error {
+ if r.err != nil {
+ return r.err
+ }
+ r.tlsRpt = append(r.tlsRpt, report)
+ return nil
+}
+
+// gzipped compresses a payload the way a reporter does before attaching it.
+func gzipped(t *testing.T, payload string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ if _, err := gz.Write([]byte(payload)); err != nil {
+ t.Fatal(err)
+ }
+ if err := gz.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return buf.Bytes()
+}
+
+// messageWithAttachment builds the shape a reporter actually sends: a short
+// human-readable part plus the report as a base64 attachment.
+func messageWithAttachment(filename, contentType string, content []byte) []byte {
+ encoded := base64.StdEncoding.EncodeToString(content)
+ var b bytes.Buffer
+ b.WriteString("From: noreply-dmarc-support@google.com\r\n")
+ b.WriteString("Subject: Report Domain: example.test\r\n")
+ b.WriteString("MIME-Version: 1.0\r\n")
+ b.WriteString("Content-Type: multipart/mixed; boundary=\"sep\"\r\n\r\n")
+ b.WriteString("--sep\r\n")
+ b.WriteString("Content-Type: text/plain; charset=utf-8\r\n\r\n")
+ b.WriteString("This is an aggregate report.\r\n")
+ b.WriteString("--sep\r\n")
+ b.WriteString("Content-Type: " + contentType + "; name=\"" + filename + "\"\r\n")
+ b.WriteString("Content-Disposition: attachment; filename=\"" + filename + "\"\r\n")
+ b.WriteString("Content-Transfer-Encoding: base64\r\n\r\n")
+ for len(encoded) > 76 {
+ b.WriteString(encoded[:76] + "\r\n")
+ encoded = encoded[76:]
+ }
+ b.WriteString(encoded + "\r\n")
+ b.WriteString("--sep--\r\n")
+ return b.Bytes()
+}
+
+const tlsRptJSON = `{"organization-name":"Google Inc.",` +
+ `"date-range":{"start-datetime":"2026-08-19T00:00:00Z","end-datetime":"2026-08-19T23:59:59Z"},` +
+ `"report-id":"2026-08-19T00:00:00Z_example.test",` +
+ `"policies":[{"policy":{"policy-type":"no-policy-found","policy-domain":"example.test"},` +
+ `"summary":{"total-successful-session-count":2,"total-failure-session-count":0}}]}`
+
+// The exact filename Google uses, which is what arrives in the inbox today.
+const googleTlsRptFilename = "google.com!example.test!1787097600!1787183999!001.json.gz"
+
+func newReportIngestor(repo *recordingReportRepo) *IngestDeliveredReportsUseCase {
+ return NewIngestDeliveredReportsUseCase(NewIngestReportsUseCase(repo))
+}
+
+func TestTlsRptReportAddressedToTlsRptIsIngested(t *testing.T) {
+ repo := &recordingReportRepo{}
+ uc := newReportIngestor(repo)
+
+ payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
+
+ outcome, err := uc.Ingest(context.Background(), []string{"tlsrpt@example.test"}, payload)
+ if err != nil {
+ t.Fatalf("ingest: %v", err)
+ }
+ if outcome.Kind != ReportKindTlsRpt {
+ t.Errorf("kind is %q, want tlsrpt", outcome.Kind)
+ }
+ if len(repo.tlsRpt) != 1 {
+ t.Fatalf("want 1 stored report, got %d", len(repo.tlsRpt))
+ }
+ if repo.tlsRpt[0].Domain != "example.test" {
+ t.Errorf("stored domain is %q", repo.tlsRpt[0].Domain)
+ }
+}
+
+func TestDmarcReportAddressedToDmarcIsIngested(t *testing.T) {
+ repo := &recordingReportRepo{}
+ uc := newReportIngestor(repo)
+
+ payload := messageWithAttachment(
+ "google.com!example.test!1787097600!1787183999.xml.gz",
+ "application/gzip", gzipped(t, dmarcAggregateXMLFixture))
+
+ outcome, err := uc.Ingest(context.Background(), []string{"dmarc@example.test"}, payload)
+ if err != nil {
+ t.Fatalf("ingest: %v", err)
+ }
+ if outcome.Kind != ReportKindDmarc {
+ t.Errorf("kind is %q, want dmarc", outcome.Kind)
+ }
+ if len(repo.dmarc) != 1 {
+ t.Fatalf("want 1 stored report, got %d", len(repo.dmarc))
+ }
+}
+
+// The address decides, not the subject or the filename - both of which the
+// sender chooses freely. Ordinary mail must pass through untouched.
+func TestOrdinaryMailIsNotTreatedAsAReport(t *testing.T) {
+ repo := &recordingReportRepo{}
+ uc := newReportIngestor(repo)
+
+ payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
+
+ outcome, err := uc.Ingest(context.Background(), []string{"lucas@example.test"}, payload)
+ if err != nil {
+ t.Fatalf("ingest: %v", err)
+ }
+ if outcome.Kind != ReportKindNone {
+ t.Errorf("ordinary mail was classified as %q", outcome.Kind)
+ }
+ if len(repo.tlsRpt) != 0 || len(repo.dmarc) != 0 {
+ t.Error("a report was stored from mail addressed to a person")
+ }
+}
+
+// A report this server cannot parse is still a report: it must be recognised,
+// so it files with the others, and it must never fail the delivery. Returning
+// an error here would have the reporter retry the same broken payload forever.
+func TestUnparseableReportIsRecognisedButNeverFailsDelivery(t *testing.T) {
+ repo := &recordingReportRepo{}
+ uc := newReportIngestor(repo)
+
+ payload := messageWithAttachment("broken.json.gz", "application/gzip", []byte("not a report at all"))
+
+ outcome, err := uc.Ingest(context.Background(), []string{"tlsrpt@example.test"}, payload)
+ if err != nil {
+ t.Fatalf("a malformed report must not fail the delivery: %v", err)
+ }
+ if outcome.Kind != ReportKindTlsRpt {
+ t.Errorf("kind is %q; the address still says what this message is", outcome.Kind)
+ }
+ if len(repo.tlsRpt) != 0 {
+ t.Error("an unparseable payload was stored")
+ }
+}
+
+// A storage failure is this server's problem, not the reporter's, and must not
+// turn into an SMTP error either.
+func TestStorageFailureDoesNotFailDelivery(t *testing.T) {
+ repo := &recordingReportRepo{err: errors.New("database is down")}
+ uc := newReportIngestor(repo)
+
+ payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
+
+ if _, err := uc.Ingest(context.Background(), []string{"tlsrpt@example.test"}, payload); err != nil {
+ t.Errorf("a storage failure must not fail the delivery: %v", err)
+ }
+}
+
+// Reporters vary the local part: dmarc-reports@, tlsrpt@, and the aliases this
+// server creates. Matching is on the local part, case-insensitively.
+func TestReportAddressMatchingIsCaseInsensitive(t *testing.T) {
+ for _, address := range []string{"TLSRPT@example.test", "TlsRpt@Example.Test"} {
+ if got := reportKindForAddress(address); got != ReportKindTlsRpt {
+ t.Errorf("%s classified as %q, want tlsrpt", address, got)
+ }
+ }
+ if got := reportKindForAddress("DMARC@example.test"); got != ReportKindDmarc {
+ t.Errorf("DMARC@ classified as %q", got)
+ }
+}
+
+// A report with no attachment at all - some reporters inline small ones - must
+// still not fail, and must still be recognised by its address.
+func TestReportWithNoAttachmentDoesNotFail(t *testing.T) {
+ repo := &recordingReportRepo{}
+ uc := newReportIngestor(repo)
+
+ payload := []byte("From: a@b.test\r\nSubject: hi\r\n\r\nno attachment here\r\n")
+ outcome, err := uc.Ingest(context.Background(), []string{"tlsrpt@example.test"}, payload)
+ if err != nil {
+ t.Fatalf("ingest: %v", err)
+ }
+ if outcome.Kind != ReportKindTlsRpt {
+ t.Errorf("kind is %q", outcome.Kind)
+ }
+}
+
+// A DMARC aggregate report, trimmed to the fields the parser reads.
+const dmarcAggregateXMLFixture = `
+
+
+ google.com
+ 987654321
+ 17870976001787183999
+
+ example.testquarantine
+
+
+ 203.0.113.10
+ 1
+ nonepasspass
+
+ example.test
+
+`
+
+// --- the delivery path ---------------------------------------------------
+
+// A report must be ingested and filed out of the way in one delivery, without
+// the address ever having to be a real mailbox the user reads.
+func TestDeliveredReportIsIngestedAndFiledInReports(t *testing.T) {
+ repo := &recordingReportRepo{}
+ messages := &recordingMessageRepository{}
+ uc := NewProcessInboundEmailUseCase(nil, &capturingBlobStorage{}, messages)
+ uc.SetReportIngestor(newReportIngestor(repo))
+
+ payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
+ if err := uc.Handle(context.Background(), ProcessInboundEmailInput{
+ Sender: "smtp-tls-reporting@google.com",
+ Recipients: []port.MailboxRecord{{ID: uuid.New()}},
+ RecipientAddresses: []string{"tlsrpt@example.test"},
+ Body: bytes.NewReader(payload),
+ }); err != nil {
+ t.Fatalf("Handle: %v", err)
+ }
+
+ if len(repo.tlsRpt) != 1 {
+ t.Fatalf("the report was not ingested: %d stored", len(repo.tlsRpt))
+ }
+ if len(messages.persisted) != 1 {
+ t.Fatalf("want the message filed once, got %d", len(messages.persisted))
+ }
+ filed := messages.persisted[0]
+ if filed.TargetFolderName != ReportsFolderName {
+ t.Errorf("the report was filed in %q, want %q", filed.TargetFolderName, ReportsFolderName)
+ }
+ // It is machine-read mail; raising an unread badge for it would recreate
+ // the inbox noise this exists to remove.
+ if !filed.AlreadySeen {
+ t.Error("the filed report was counted as unread mail")
+ }
+}
+
+// Ordinary mail must reach the inbox exactly as before.
+func TestOrdinaryMailStillReachesTheInbox(t *testing.T) {
+ repo := &recordingReportRepo{}
+ messages := &recordingMessageRepository{}
+ uc := NewProcessInboundEmailUseCase(nil, &capturingBlobStorage{}, messages)
+ uc.SetReportIngestor(newReportIngestor(repo))
+
+ if err := uc.Handle(context.Background(), ProcessInboundEmailInput{
+ Sender: "friend@example.test",
+ Recipients: []port.MailboxRecord{{ID: uuid.New()}},
+ RecipientAddresses: []string{"lucas@example.test"},
+ Body: bytes.NewBufferString("Subject: hello\r\n\r\nhi"),
+ }); err != nil {
+ t.Fatalf("Handle: %v", err)
+ }
+
+ filed := messages.persisted[0]
+ if filed.TargetFolderName == ReportsFolderName {
+ t.Error("ordinary mail was filed as a report")
+ }
+ if filed.AlreadySeen {
+ t.Error("ordinary mail was filed as already read")
+ }
+}
+
+// With no ingestor configured, delivery must behave exactly as it did before
+// this existed - reports included.
+func TestDeliveryIsUnchangedWithoutAnIngestor(t *testing.T) {
+ messages := &recordingMessageRepository{}
+ uc := NewProcessInboundEmailUseCase(nil, &capturingBlobStorage{}, messages)
+
+ payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
+ if err := uc.Handle(context.Background(), ProcessInboundEmailInput{
+ Sender: "smtp-tls-reporting@google.com",
+ Recipients: []port.MailboxRecord{{ID: uuid.New()}},
+ RecipientAddresses: []string{"tlsrpt@example.test"},
+ Body: bytes.NewReader(payload),
+ }); err != nil {
+ t.Fatalf("Handle: %v", err)
+ }
+ if messages.persisted[0].TargetFolderName == ReportsFolderName {
+ t.Error("a report was routed with no ingestor configured")
+ }
+}
diff --git a/services/protocols/internal/application/usecase/process_inbound_email.go b/services/protocols/internal/application/usecase/process_inbound_email.go
index a789b47..9510692 100644
--- a/services/protocols/internal/application/usecase/process_inbound_email.go
+++ b/services/protocols/internal/application/usecase/process_inbound_email.go
@@ -65,12 +65,20 @@ type ProcessInboundEmailUseCase struct {
messages port.InboundMessageRepository
folders port.ImapFolderRepository
trackerManager *MailboxTrackerManager
+ reportIngestor *IngestDeliveredReportsUseCase
}
func NewProcessInboundEmailUseCase(mailboxes port.MailboxRepository, blobs port.BlobStorage, messages port.InboundMessageRepository) *ProcessInboundEmailUseCase {
return &ProcessInboundEmailUseCase{mailboxes: mailboxes, blobs: blobs, messages: messages}
}
+// SetReportIngestor enables reading DMARC and TLS-RPT reports out of the mail
+// that carries them. Optional: with no ingestor a report is delivered like any
+// other message, which is what happened before this existed.
+func (uc *ProcessInboundEmailUseCase) SetReportIngestor(ingestor *IngestDeliveredReportsUseCase) {
+ uc.reportIngestor = ingestor
+}
+
func (uc *ProcessInboundEmailUseCase) SetScanner(scanner port.ContentScanner) {
uc.scanner = scanner
}
@@ -234,6 +242,26 @@ func (uc *ProcessInboundEmailUseCase) Handle(ctx context.Context, input ProcessI
}
}
+ // Reports are read here, after the scan has had its say and before the
+ // message is filed, because this is where the recipient list and the
+ // finished payload are both in hand.
+ //
+ // A report that fails to parse is still routed to Reports: the address it
+ // was sent to is what makes it one, and leaving a broken report in the
+ // inbox instead would put back exactly the noise this removes. The bytes
+ // are kept either way, so it can be re-ingested once the cause is fixed.
+ alreadySeen := false
+ if uc.reportIngestor != nil {
+ outcome, err := uc.reportIngestor.Ingest(ctx, input.RecipientAddresses, payload)
+ if err != nil {
+ return err
+ }
+ if outcome.Kind != ReportKindNone {
+ targetFolder = ReportsFolderName
+ alreadySeen = true
+ }
+ }
+
blob, err := uc.blobs.Store(ctx, bytes.NewReader(payload))
if err != nil {
return err
@@ -261,6 +289,7 @@ func (uc *ProcessInboundEmailUseCase) Handle(ctx context.Context, input ProcessI
FromDisplayName: headers.FromDisplayName,
MessageIDHeader: headers.MessageID,
HasAttachments: headers.HasAttachments,
+ AlreadySeen: alreadySeen,
SPFResult: authResult.SPF,
DKIMResult: authResult.DKIM,
DMARCResult: authResult.DMARC,
diff --git a/services/protocols/internal/domain/entity/dmarc_report.go b/services/protocols/internal/domain/entity/dmarc_report.go
index fe37e28..e3d69f4 100644
--- a/services/protocols/internal/domain/entity/dmarc_report.go
+++ b/services/protocols/internal/domain/entity/dmarc_report.go
@@ -1,6 +1,7 @@
package entity
import (
+ "archive/zip"
"bytes"
"compress/gzip"
"encoding/xml"
@@ -55,10 +56,16 @@ type dmarcXmlReport struct {
} `xml:"record"`
}
-// ParseDmarcXmlReport parses raw XML or gzipped XML DMARC aggregate report.
+// ParseDmarcXmlReport parses a DMARC aggregate report: raw XML, gzipped XML,
+// or XML inside a zip archive.
+//
+// Zip matters as much as gzip. Microsoft and Yahoo send .zip, and between them
+// they account for a large share of the reports any domain receives; handling
+// gzip alone meant those reports never parsed at all.
func ParseDmarcXmlReport(payload []byte) (*DmarcReport, error) {
data := payload
- if isGzip(payload) {
+ switch {
+ case isGzip(payload):
gz, err := gzip.NewReader(bytes.NewReader(payload))
if err != nil {
return nil, fmt.Errorf("gzip reader: %w", err)
@@ -69,6 +76,12 @@ func ParseDmarcXmlReport(payload []byte) (*DmarcReport, error) {
return nil, fmt.Errorf("decompress gzip: %w", err)
}
data = decompressed
+ case isZip(payload):
+ decompressed, err := firstFileInZip(payload)
+ if err != nil {
+ return nil, err
+ }
+ data = decompressed
}
var parsed dmarcXmlReport
@@ -101,3 +114,41 @@ func ParseDmarcXmlReport(payload []byte) (*DmarcReport, error) {
func isGzip(payload []byte) bool {
return len(payload) >= 2 && payload[0] == 0x1f && payload[1] == 0x8b
}
+
+// isZip reports whether the payload begins with a local file header. The
+// empty-archive and spanned-archive signatures are deliberately not accepted:
+// neither carries a report.
+func isZip(payload []byte) bool {
+ return len(payload) >= 4 && payload[0] == 'P' && payload[1] == 'K' &&
+ payload[2] == 0x03 && payload[3] == 0x04
+}
+
+// firstFileInZip returns the contents of the first regular file in the
+// archive. An aggregate report is one XML document per archive, so there is
+// nothing to choose between; directories are skipped so an archive that
+// carries one cannot be mistaken for an empty one.
+func firstFileInZip(payload []byte) ([]byte, error) {
+ reader, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
+ if err != nil {
+ return nil, fmt.Errorf("zip reader: %w", err)
+ }
+ for _, file := range reader.File {
+ if file.FileInfo().IsDir() {
+ continue
+ }
+ handle, err := file.Open()
+ if err != nil {
+ return nil, fmt.Errorf("open %s in zip: %w", file.Name, err)
+ }
+ defer handle.Close()
+ // Bounded so a zip bomb cannot exhaust this process: an aggregate
+ // report is XML in the tens of kilobytes, and 64 MiB is far past any
+ // legitimate one.
+ decompressed, err := io.ReadAll(io.LimitReader(handle, 64<<20))
+ if err != nil {
+ return nil, fmt.Errorf("decompress %s in zip: %w", file.Name, err)
+ }
+ return decompressed, nil
+ }
+ return nil, fmt.Errorf("zip archive contains no report")
+}
diff --git a/services/protocols/internal/domain/entity/dmarc_zip_test.go b/services/protocols/internal/domain/entity/dmarc_zip_test.go
new file mode 100644
index 0000000..552cc27
--- /dev/null
+++ b/services/protocols/internal/domain/entity/dmarc_zip_test.go
@@ -0,0 +1,85 @@
+package entity
+
+import (
+ "archive/zip"
+ "bytes"
+ "testing"
+)
+
+// A real DMARC aggregate report, trimmed to the fields the parser reads.
+const dmarcAggregateXML = `
+
+
+ Yahoo
+ 1234567890
+ 17870976001787183999
+
+
+ example.test
+ quarantine
+
+
+
+ 203.0.113.10
+ 3
+ nonepasspass
+
+ example.test
+
+`
+
+// Microsoft and Yahoo send aggregate reports as .zip, not .gz. The parser
+// handled gzip only, so every report from those two failed to parse - and
+// between them they are a large share of the reports any domain receives.
+func TestDmarcReportParsesZippedPayload(t *testing.T) {
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ f, err := zw.Create("example.test!1787097600!1787183999.xml")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := f.Write([]byte(dmarcAggregateXML)); err != nil {
+ t.Fatal(err)
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ report, err := ParseDmarcXmlReport(buf.Bytes())
+ if err != nil {
+ t.Fatalf("parse zipped dmarc report: %v", err)
+ }
+ if report.OrgName != "Yahoo" {
+ t.Errorf("org name is %q, want Yahoo", report.OrgName)
+ }
+ if report.Domain != "example.test" {
+ t.Errorf("domain is %q, want example.test", report.Domain)
+ }
+ if len(report.Records) != 1 {
+ t.Fatalf("want 1 record, got %d", len(report.Records))
+ }
+}
+
+// Plain XML must keep working exactly as before.
+func TestDmarcReportStillParsesPlainXml(t *testing.T) {
+ report, err := ParseDmarcXmlReport([]byte(dmarcAggregateXML))
+ if err != nil {
+ t.Fatalf("parse plain xml: %v", err)
+ }
+ if report.OrgName != "Yahoo" {
+ t.Errorf("org name is %q", report.OrgName)
+ }
+}
+
+// An empty archive has no report in it and must be reported as such rather
+// than panicking on a missing first file.
+func TestDmarcReportRejectsEmptyZip(t *testing.T) {
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ if err := zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := ParseDmarcXmlReport(buf.Bytes()); err == nil {
+ t.Error("an empty zip should not parse as a report")
+ }
+}
diff --git a/services/protocols/internal/infrastructure/postgres/delivered_report_store.go b/services/protocols/internal/infrastructure/postgres/delivered_report_store.go
new file mode 100644
index 0000000..4985550
--- /dev/null
+++ b/services/protocols/internal/infrastructure/postgres/delivered_report_store.go
@@ -0,0 +1,156 @@
+package postgres
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5/pgxpool"
+
+ "lambdamail/protocols/internal/application/port"
+)
+
+// DeliveredReportStore reaches reports that were delivered before automatic
+// ingestion existed and are still filed in a mailbox.
+type DeliveredReportStore struct {
+ pool *pgxpool.Pool
+}
+
+func NewDeliveredReportStore(pool *pgxpool.Pool) *DeliveredReportStore {
+ return &DeliveredReportStore{pool: pool}
+}
+
+// reportRecipientPattern matches the local parts the report aliases use. It
+// mirrors reportLocalParts in the use case; both exist because this one has to
+// be expressible in SQL.
+const reportRecipientPattern = `^(dmarc|tlsrpt)@`
+
+// ListUningestedReports returns stored messages addressed to a report alias
+// that are not already filed in Reports.
+//
+// Being outside Reports is what "not yet ingested" means here: the delivery
+// path files every report it reads into that folder, so anything addressed to
+// a report alias and still sitting elsewhere predates ingestion.
+func (s *DeliveredReportStore) ListUningestedReports(
+ ctx context.Context, limit int,
+) ([]port.StoredReportMessage, error) {
+ if limit <= 0 || limit > 5000 {
+ limit = 5000
+ }
+
+ rows, err := s.pool.Query(ctx, `
+ SELECT e.id, e.blob_id, recipient
+ FROM email_messages e
+ JOIN folders f ON f.id = e.folder_id
+ CROSS JOIN LATERAL unnest(e.recipient_addresses) AS recipient
+ WHERE e.expunged_at IS NULL
+ AND LOWER(f.name) <> LOWER($1)
+ AND recipient ~* $2
+ ORDER BY e.received_at
+ LIMIT $3
+ `, reportsFolderName, reportRecipientPattern, limit)
+ if err != nil {
+ return nil, fmt.Errorf("list delivered reports: %w", err)
+ }
+ defer rows.Close()
+
+ out := []port.StoredReportMessage{}
+ for rows.Next() {
+ var msg port.StoredReportMessage
+ if err := rows.Scan(&msg.MessageID, &msg.BlobID, &msg.RecipientAddress); err != nil {
+ return nil, err
+ }
+ out = append(out, msg)
+ }
+ return out, rows.Err()
+}
+
+// reportsFolderName is duplicated from the use case rather than imported: the
+// infrastructure layer must not depend on the application layer.
+const reportsFolderName = "Reports"
+
+// MoveToReportsFolder files a backfilled message out of the inbox, keeping the
+// counters on both folders in step.
+//
+// The message keeps its identity and its blob; only the folder and the UID
+// change, since a UID means nothing outside one folder.
+func (s *DeliveredReportStore) MoveToReportsFolder(ctx context.Context, messageID uuid.UUID) error {
+ tx, err := s.pool.Begin(ctx)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback(ctx)
+
+ var mailboxID uuid.UUID
+ var sourceFolderID string
+ var wasUnread bool
+ err = tx.QueryRow(ctx, `
+ SELECT e.mailbox_id, e.folder_id::text,
+ NOT EXISTS (SELECT 1 FROM message_flags mf
+ WHERE mf.message_id = e.id
+ AND mf.received_at = e.received_at
+ AND mf.flag = '\Seen')
+ FROM email_messages e
+ WHERE e.id = $1 AND e.expunged_at IS NULL
+ `, messageID).Scan(&mailboxID, &sourceFolderID, &wasUnread)
+ if err != nil {
+ return fmt.Errorf("find message %s: %w", messageID, err)
+ }
+
+ var targetFolderID string
+ var targetUID int64
+ err = tx.QueryRow(ctx, `
+ SELECT id::text, uid_next FROM folders
+ WHERE mailbox_id = $1 AND LOWER(name) = LOWER($2)
+ LIMIT 1
+ FOR UPDATE
+ `, mailboxID, reportsFolderName).Scan(&targetFolderID, &targetUID)
+ if err != nil {
+ return fmt.Errorf("find the %s folder for mailbox %s: %w", reportsFolderName, mailboxID, err)
+ }
+ if targetFolderID == sourceFolderID {
+ return nil
+ }
+
+ var modseq int64
+ if err := tx.QueryRow(ctx, `
+ UPDATE folders SET uid_next = uid_next + 1, highest_modseq = highest_modseq + 1,
+ total_count = total_count + 1
+ WHERE id = $1
+ RETURNING highest_modseq
+ `, targetFolderID).Scan(&modseq); err != nil {
+ return err
+ }
+
+ if _, err := tx.Exec(ctx, `
+ UPDATE email_messages SET folder_id = $2, uid = $3, modseq = $4 WHERE id = $1
+ `, messageID, targetFolderID, targetUID, modseq); err != nil {
+ return err
+ }
+
+ // A backfilled report is machine-read mail: marking it seen is what stops
+ // it raising an unread badge in its new home.
+ if _, err := tx.Exec(ctx, `
+ INSERT INTO message_flags (message_id, received_at, flag)
+ SELECT e.id, e.received_at, '\Seen' FROM email_messages e WHERE e.id = $1
+ ON CONFLICT DO NOTHING
+ `, messageID); err != nil {
+ return err
+ }
+
+ unreadDelta := 0
+ if wasUnread {
+ unreadDelta = -1
+ }
+ if _, err := tx.Exec(ctx, `
+ UPDATE folders
+ SET total_count = GREATEST(0, total_count - 1),
+ unread_count = GREATEST(0, unread_count + $2),
+ highest_modseq = highest_modseq + 1
+ WHERE id = $1
+ `, sourceFolderID, unreadDelta); err != nil {
+ return err
+ }
+
+ return tx.Commit(ctx)
+}
diff --git a/services/protocols/internal/infrastructure/postgres/report_ingestion_test.go b/services/protocols/internal/infrastructure/postgres/report_ingestion_test.go
new file mode 100644
index 0000000..07dc941
--- /dev/null
+++ b/services/protocols/internal/infrastructure/postgres/report_ingestion_test.go
@@ -0,0 +1,355 @@
+package postgres
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5/pgxpool"
+
+ "lambdamail/protocols/internal/application/port"
+ "lambdamail/protocols/internal/application/usecase"
+ "lambdamail/protocols/internal/domain/entity"
+ "lambdamail/protocols/internal/infrastructure/diskstorage"
+)
+
+// End to end over a real database: a report arrives as mail, is parsed into
+// the report tables, and is filed into Reports without raising an unread badge.
+
+// The report Google actually sends, with the domain changed.
+const liveTlsRptReport = `{"organization-name":"Google Inc.",` +
+ `"date-range":{"start-datetime":"2026-08-19T00:00:00Z","end-datetime":"2026-08-19T23:59:59Z"},` +
+ `"contact-info":"smtp-tls-reporting@google.com","report-id":"2026-08-19T00:00:00Z_example.test",` +
+ `"policies":[{"policy":{"policy-type":"no-policy-found","policy-domain":"example.test"},` +
+ `"summary":{"total-successful-session-count":2,"total-failure-session-count":0}}]}`
+
+func reportMail(t *testing.T, filename string, content []byte) []byte {
+ t.Helper()
+ var gzBuf bytes.Buffer
+ gz := gzip.NewWriter(&gzBuf)
+ if _, err := gz.Write(content); err != nil {
+ t.Fatal(err)
+ }
+ if err := gz.Close(); err != nil {
+ t.Fatal(err)
+ }
+ encoded := base64.StdEncoding.EncodeToString(gzBuf.Bytes())
+
+ var b bytes.Buffer
+ b.WriteString("From: smtp-tls-reporting@google.com\r\n")
+ b.WriteString("Subject: Report Domain: example.test\r\n")
+ b.WriteString("MIME-Version: 1.0\r\n")
+ b.WriteString("Content-Type: multipart/mixed; boundary=\"sep\"\r\n\r\n")
+ b.WriteString("--sep\r\n")
+ b.WriteString("Content-Type: text/plain; charset=utf-8\r\n\r\nReport attached.\r\n")
+ b.WriteString("--sep\r\n")
+ b.WriteString("Content-Type: application/gzip; name=\"" + filename + "\"\r\n")
+ b.WriteString("Content-Disposition: attachment; filename=\"" + filename + "\"\r\n")
+ b.WriteString("Content-Transfer-Encoding: base64\r\n\r\n")
+ b.WriteString(encoded + "\r\n")
+ b.WriteString("--sep--\r\n")
+ return b.Bytes()
+}
+
+func TestDeliveredTlsRptReportIsIngestedAndFiled(t *testing.T) {
+ pool := testPool(t)
+ ctx := context.Background()
+ mailboxID := seedMailbox(t, pool)
+
+ // The Reports folder migration 0008 adds for every mailbox.
+ if _, err := pool.Exec(ctx,
+ `INSERT INTO folders (mailbox_id, name, special_use) VALUES ($1, 'Reports', NULL)`,
+ uuid.MustParse(mailboxID)); err != nil {
+ t.Fatalf("seed Reports folder: %v", err)
+ }
+
+ reportRepo := NewReportRepository(pool)
+ inbound := usecase.NewProcessInboundEmailUseCase(
+ nil, &dbBlobStore{pool: pool, dir: t.TempDir()}, NewInboundMessageRepository(pool))
+ inbound.SetReportIngestor(
+ usecase.NewIngestDeliveredReportsUseCase(usecase.NewIngestReportsUseCase(reportRepo)))
+
+ reportID := "delivered-" + uuid.NewString()
+ payload := reportMail(t,
+ "google.com!example.test!1787097600!1787183999!001.json.gz",
+ []byte(strings.Replace(liveTlsRptReport,
+ "2026-08-19T00:00:00Z_example.test", reportID, 1)))
+ t.Cleanup(func() {
+ _, _ = pool.Exec(context.Background(),
+ `DELETE FROM tls_rpt_reports WHERE report_id = $1`, reportID)
+ })
+
+ if err := inbound.Handle(ctx, usecase.ProcessInboundEmailInput{
+ Sender: "smtp-tls-reporting@google.com",
+ Recipients: []port.MailboxRecord{{ID: uuid.MustParse(mailboxID)}},
+ RecipientAddresses: []string{"tlsrpt@example.test"},
+ Body: bytes.NewReader(payload),
+ }); err != nil {
+ t.Fatalf("Handle: %v", err)
+ }
+
+ // The report reached the table the admin console reads.
+ var stored int
+ if err := pool.QueryRow(ctx,
+ `SELECT COUNT(*) FROM tls_rpt_reports WHERE report_id = $1`, reportID).Scan(&stored); err != nil {
+ t.Fatalf("count stored reports: %v", err)
+ }
+ if stored != 1 {
+ t.Errorf("the report was not stored: %d rows", stored)
+ }
+
+ // And the message went to Reports, silently.
+ webmail := NewWebmailRepository(pool)
+ folders, err := webmail.ListFolders(ctx, mailboxID)
+ if err != nil {
+ t.Fatalf("list folders: %v", err)
+ }
+ for _, f := range folders {
+ switch f.Name {
+ case "Reports":
+ if f.TotalCount != 1 {
+ t.Errorf("Reports holds %d messages, want 1", f.TotalCount)
+ }
+ if f.UnreadCount != 0 {
+ t.Errorf("Reports shows %d unread; a machine-read report raises no badge", f.UnreadCount)
+ }
+ case "INBOX":
+ if f.TotalCount != 0 {
+ t.Errorf("the report landed in the inbox: %d messages", f.TotalCount)
+ }
+ }
+ }
+}
+
+// dbBlobStore writes a real message_blobs row so the delivery's foreign keys
+// hold, without needing the disk storage driver in a database test.
+type dbBlobStore struct {
+ pool *pgxpool.Pool
+ // dir is where the bytes actually land, so the real disk reader can read
+ // them back. Pointing storage_path at a file that does not exist would
+ // leave the read path untested.
+ dir string
+}
+
+func (s *dbBlobStore) Store(ctx context.Context, r io.Reader) (port.BlobRef, error) {
+ body, err := io.ReadAll(r)
+ if err != nil {
+ return port.BlobRef{}, err
+ }
+ sum := sha256.Sum256(body)
+ digest := hex.EncodeToString(sum[:])
+ // Deduplicated on the digest, as the real storage drivers are: identical
+ // bytes are one blob, and a second delivery of the same report reuses it.
+ id := uuid.New()
+ path := filepath.Join(s.dir, id.String()+".eml")
+ if err := os.WriteFile(path, body, 0o600); err != nil {
+ return port.BlobRef{}, err
+ }
+ if err := s.pool.QueryRow(ctx, `
+ INSERT INTO message_blobs (id, content_sha256, storage_driver, storage_path, size_bytes, ref_count)
+ VALUES ($1, $2, 'local', $3, $4, 0)
+ ON CONFLICT (content_sha256) DO UPDATE SET storage_path = EXCLUDED.storage_path
+ RETURNING id`,
+ id, digest, path, len(body)).Scan(&id); err != nil {
+ return port.BlobRef{}, err
+ }
+ return port.BlobRef{ID: id, SHA256: digest, SizeBytes: int64(len(body))}, nil
+}
+
+// A reporter that retries - or an SMTP retry after a slow commit - delivers
+// the same report twice. Automatic ingestion makes that routine, where before
+// reports were only ever written by hand through the API, so the same report
+// arriving again must not double every number the console shows.
+func TestIngestingTheSameReportTwiceStoresItOnce(t *testing.T) {
+ pool := testPool(t)
+ ctx := context.Background()
+ repo := NewReportRepository(pool)
+
+ reportID := "retry-" + uuid.NewString()
+ report := func() *entity.TlsRptReport {
+ return &entity.TlsRptReport{
+ OrganizationName: "Google Inc.",
+ ReportID: reportID,
+ Domain: "example.test",
+ DateRangeBegin: time.Now().Add(-24 * time.Hour),
+ DateRangeEnd: time.Now(),
+ Policies: []entity.TlsRptPolicy{
+ {PolicyType: "no-policy-found", SuccessCount: 2, FailureCount: 0},
+ },
+ }
+ }
+ t.Cleanup(func() {
+ _, _ = pool.Exec(context.Background(),
+ `DELETE FROM tls_rpt_reports WHERE report_id = $1`, reportID)
+ })
+
+ if err := repo.SaveTlsRptReport(ctx, report()); err != nil {
+ t.Fatalf("first save: %v", err)
+ }
+ if err := repo.SaveTlsRptReport(ctx, report()); err != nil {
+ t.Fatalf("a redelivered report must not error: %v", err)
+ }
+
+ var headers, policies int
+ if err := pool.QueryRow(ctx,
+ `SELECT COUNT(*) FROM tls_rpt_reports WHERE report_id = $1`, reportID).Scan(&headers); err != nil {
+ t.Fatal(err)
+ }
+ if headers != 1 {
+ t.Errorf("the same report is stored %d times", headers)
+ }
+ if err := pool.QueryRow(ctx, `
+ SELECT COUNT(*) FROM tls_rpt_report_policies p
+ JOIN tls_rpt_reports r ON r.id = p.report_id
+ WHERE r.report_id = $1`, reportID).Scan(&policies); err != nil {
+ t.Fatal(err)
+ }
+ if policies != 1 {
+ t.Errorf("the report's policies are stored %d times", policies)
+ }
+}
+
+// The same, for DMARC: its unique key includes domain_id, which this path
+// never fills, and in Postgres a NULL never conflicts - so the key it had did
+// not in fact prevent a duplicate.
+func TestIngestingTheSameDmarcReportTwiceStoresItOnce(t *testing.T) {
+ pool := testPool(t)
+ ctx := context.Background()
+ repo := NewReportRepository(pool)
+
+ reportID := "retry-" + uuid.NewString()
+ report := func() *entity.DmarcReport {
+ return &entity.DmarcReport{
+ OrgName: "google.com",
+ ReportID: reportID,
+ Domain: "example.test",
+ DateRangeBegin: time.Now().Add(-24 * time.Hour),
+ DateRangeEnd: time.Now(),
+ Records: []entity.DmarcRecord{
+ {SourceIP: "203.0.113.10", Count: 3, Disposition: "none", DKIMResult: "pass", SPFResult: "pass"},
+ },
+ }
+ }
+ t.Cleanup(func() {
+ _, _ = pool.Exec(context.Background(),
+ `DELETE FROM dmarc_reports WHERE report_id = $1`, reportID)
+ })
+
+ if err := repo.SaveDmarcReport(ctx, report()); err != nil {
+ t.Fatalf("first save: %v", err)
+ }
+ if err := repo.SaveDmarcReport(ctx, report()); err != nil {
+ t.Fatalf("a redelivered report must not error: %v", err)
+ }
+
+ var headers int
+ if err := pool.QueryRow(ctx,
+ `SELECT COUNT(*) FROM dmarc_reports WHERE report_id = $1`, reportID).Scan(&headers); err != nil {
+ t.Fatal(err)
+ }
+ if headers != 1 {
+ t.Errorf("the same DMARC report is stored %d times", headers)
+ }
+}
+
+// The backfill, over a real database: a report that was delivered to the
+// inbox before ingestion existed is parsed after the fact and filed away.
+func TestBackfillParsesAReportAlreadySittingInTheInbox(t *testing.T) {
+ pool := testPool(t)
+ ctx := context.Background()
+ mailboxID := seedMailbox(t, pool)
+
+ if _, err := pool.Exec(ctx,
+ `INSERT INTO folders (mailbox_id, name, special_use) VALUES ($1, 'Reports', NULL)`,
+ uuid.MustParse(mailboxID)); err != nil {
+ t.Fatalf("seed Reports folder: %v", err)
+ }
+
+ // Delivered the old way: straight into the inbox, unread, never parsed.
+ reportID := "backfill-" + uuid.NewString()
+ body := `{"organization-name":"Google Inc.",` +
+ `"date-range":{"start-datetime":"2026-08-19T00:00:00Z","end-datetime":"2026-08-19T23:59:59Z"},` +
+ `"report-id":"` + reportID + `",` +
+ `"policies":[{"policy":{"policy-type":"sts","policy-domain":"example.test"},` +
+ `"summary":{"total-successful-session-count":5,"total-failure-session-count":1}}]}`
+ payload := reportMail(t, "google.com!example.test!1!2!001.json.gz", []byte(body))
+
+ blobs := &dbBlobStore{pool: pool, dir: t.TempDir()}
+ blob, err := blobs.Store(ctx, bytes.NewReader(payload))
+ if err != nil {
+ t.Fatalf("store blob: %v", err)
+ }
+ uids, err := NewInboundMessageRepository(pool).PersistAll(ctx, []port.PersistInboundMessageInput{{
+ MailboxID: uuid.MustParse(mailboxID),
+ Blob: blob,
+ SenderAddress: "smtp-tls-reporting@google.com",
+ RecipientAddress: "tlsrpt@example.test",
+ TargetFolderName: "INBOX",
+ Subject: "Report Domain: example.test",
+ }})
+ if err != nil || len(uids) != 1 {
+ t.Fatalf("seed delivered report: %v", err)
+ }
+ t.Cleanup(func() {
+ _, _ = pool.Exec(context.Background(),
+ `DELETE FROM tls_rpt_reports WHERE report_id = $1`, reportID)
+ })
+
+ store := NewDeliveredReportStore(pool)
+ uc := usecase.NewBackfillReportsUseCase(store, diskstorage.NewLocalDiskBlobReader(pool),
+ usecase.NewIngestDeliveredReportsUseCase(
+ usecase.NewIngestReportsUseCase(NewReportRepository(pool))))
+
+ summary, err := uc.Run(ctx, 0)
+ if err != nil {
+ t.Fatalf("backfill: %v", err)
+ }
+ if summary.Ingested < 1 {
+ t.Fatalf("the backlogged report was not ingested: %+v", summary)
+ }
+
+ var stored int
+ if err := pool.QueryRow(ctx,
+ `SELECT COUNT(*) FROM tls_rpt_reports WHERE report_id = $1`, reportID).Scan(&stored); err != nil {
+ t.Fatal(err)
+ }
+ if stored != 1 {
+ t.Errorf("the report reached the table %d times, want 1", stored)
+ }
+
+ // And it left the inbox.
+ webmail := NewWebmailRepository(pool)
+ folders, _ := webmail.ListFolders(ctx, mailboxID)
+ for _, f := range folders {
+ if f.Name == "INBOX" && f.TotalCount != 0 {
+ t.Errorf("the backfilled report is still in the inbox: %d messages", f.TotalCount)
+ }
+ if f.Name == "Reports" {
+ if f.TotalCount != 1 {
+ t.Errorf("Reports holds %d messages, want 1", f.TotalCount)
+ }
+ if f.UnreadCount != 0 {
+ t.Errorf("the backfilled report raised an unread badge: %d", f.UnreadCount)
+ }
+ }
+ }
+
+ // Running it again finds nothing left to do.
+ second, err := uc.Run(ctx, 0)
+ if err != nil {
+ t.Fatalf("second run: %v", err)
+ }
+ if second.Ingested != 0 {
+ t.Errorf("a second pass re-ingested %d reports; it should find none", second.Ingested)
+ }
+}
diff --git a/services/protocols/internal/infrastructure/postgres/report_repository.go b/services/protocols/internal/infrastructure/postgres/report_repository.go
index 1ac38ac..92ce7b1 100644
--- a/services/protocols/internal/infrastructure/postgres/report_repository.go
+++ b/services/protocols/internal/infrastructure/postgres/report_repository.go
@@ -25,16 +25,35 @@ func (r *ReportRepository) SaveDmarcReport(ctx context.Context, report *entity.D
}
defer tx.Rollback(ctx)
+ // Idempotent on (org_name, report_id): the reporter's own identifier for
+ // the report. Reports are now parsed out of inbound mail automatically, so
+ // a redelivery - an SMTP retry, or a reporter simply sending again - is
+ // routine, and storing the same report twice would double every number the
+ // admin console shows.
+ //
+ // DO UPDATE rather than DO NOTHING so the existing row's id comes back and
+ // the records below can be reconciled against it; DO NOTHING returns no
+ // row at all when it conflicts.
var reportID uuid.UUID
err = tx.QueryRow(ctx, `
INSERT INTO dmarc_reports (org_name, report_id, domain, date_range_begin, date_range_end)
VALUES ($1, $2, $3, $4, $5)
+ ON CONFLICT (org_name, report_id)
+ DO UPDATE SET domain = EXCLUDED.domain
RETURNING id
`, report.OrgName, report.ReportID, report.Domain, report.DateRangeBegin, report.DateRangeEnd).Scan(&reportID)
if err != nil {
return fmt.Errorf("insert dmarc report header: %w", err)
}
+ // The records are replaced, not appended: on a redelivery the report's
+ // previous rows are still here, and adding another set would double every
+ // count derived from them.
+ if _, err := tx.Exec(ctx,
+ `DELETE FROM dmarc_report_records WHERE report_id = $1`, reportID); err != nil {
+ return fmt.Errorf("clear dmarc records: %w", err)
+ }
+
for _, rec := range report.Records {
_, err := tx.Exec(ctx, `
INSERT INTO dmarc_report_records (report_id, source_ip, count, disposition, dkim_result, spf_result, header_from)
@@ -55,16 +74,24 @@ func (r *ReportRepository) SaveTlsRptReport(ctx context.Context, report *entity.
}
defer tx.Rollback(ctx)
+ // Idempotent for the same reason as the DMARC writer above.
var reportID uuid.UUID
err = tx.QueryRow(ctx, `
INSERT INTO tls_rpt_reports (organization_name, report_id, domain, date_range_begin, date_range_end)
VALUES ($1, $2, $3, $4, $5)
+ ON CONFLICT (organization_name, report_id)
+ DO UPDATE SET domain = EXCLUDED.domain
RETURNING id
`, report.OrganizationName, report.ReportID, report.Domain, report.DateRangeBegin, report.DateRangeEnd).Scan(&reportID)
if err != nil {
return fmt.Errorf("insert tls-rpt report header: %w", err)
}
+ if _, err := tx.Exec(ctx,
+ `DELETE FROM tls_rpt_report_policies WHERE report_id = $1`, reportID); err != nil {
+ return fmt.Errorf("clear tls-rpt policies: %w", err)
+ }
+
for _, pol := range report.Policies {
_, err := tx.Exec(ctx, `
INSERT INTO tls_rpt_report_policies (report_id, policy_type, success_count, failure_count)
diff --git a/services/protocols/internal/presentation/http/events_ws_test.go b/services/protocols/internal/presentation/http/events_ws_test.go
new file mode 100644
index 0000000..f9eb556
--- /dev/null
+++ b/services/protocols/internal/presentation/http/events_ws_test.go
@@ -0,0 +1,191 @@
+package httppresentation
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/google/uuid"
+
+ "lambdamail/protocols/internal/application/port"
+ "lambdamail/protocols/internal/application/usecase"
+)
+
+// The event stream tells a browser its mailbox changed. It is the one place
+// where one account's activity is pushed out over another's connection if the
+// filtering is wrong, and none of it was covered.
+
+func newTestHub(t *testing.T, store *fakeMailStore) *EventHub {
+ t.Helper()
+ uc := usecase.NewWebmailUseCase(store, &fakeBlobReader{}, nil, nil, "mail.example.test")
+ return NewEventHub(testVerifier(interopSecret), uc)
+}
+
+// The hub matches on the aggregate's UUID string, so subscribers are keyed by
+// real UUIDs rather than the readable names the mail API fakes use.
+var (
+ mailboxOne = uuid.New()
+ mailboxTwo = uuid.New()
+)
+
+// subscribeTo registers a subscriber directly, which is what the WebSocket
+// handler does after authenticating. Driving a real upgrade would test the
+// library; this tests the routing decision, which is ours.
+func subscribeTo(hub *EventHub, mailboxID string, buffer int) *subscriber {
+ sub := &subscriber{mailboxID: mailboxID, events: make(chan EventMessage, buffer)}
+ hub.add(sub)
+ return sub
+}
+
+// The check that keeps one mailbox's activity off another's connection.
+func TestEventsReachOnlyTheirOwnMailbox(t *testing.T) {
+ hub := newTestHub(t, storeWithMail())
+ mine := subscribeTo(hub, mailboxOne.String(), 4)
+ theirs := subscribeTo(hub, mailboxTwo.String(), 4)
+
+ hub.Publish(port.OutboxEvent{
+ EventType: "EmailReceived",
+ AggregateID: mailboxOne,
+ Payload: []byte(`{"uid":7}`),
+ })
+
+ select {
+ case msg := <-mine.events:
+ if msg.Type != "EmailReceived" {
+ t.Errorf("got event type %q", msg.Type)
+ }
+ default:
+ t.Error("the mailbox's own event never arrived")
+ }
+
+ select {
+ case msg := <-theirs.events:
+ t.Errorf("another mailbox's event was delivered: %+v", msg)
+ default:
+ }
+}
+
+// A subscriber that has stopped reading must be dropped, never allowed to
+// stall the relay that every other connection shares.
+func TestASlowSubscriberIsDroppedRatherThanBlocking(t *testing.T) {
+ hub := newTestHub(t, storeWithMail())
+ // Buffer of one, then filled, so the next publish has nowhere to go.
+ stalled := subscribeTo(hub, mailboxOne.String(), 1)
+ stalled.events <- EventMessage{Type: "filler"}
+
+ done := make(chan struct{})
+ go func() {
+ hub.Publish(port.OutboxEvent{
+ EventType: "EmailReceived",
+ AggregateID: mailboxOne,
+ Payload: []byte(`{}`),
+ })
+ close(done)
+ }()
+
+ select {
+ case <-done:
+ case <-context.Background().Done():
+ t.Fatal("Publish blocked on a subscriber that was not reading")
+ }
+}
+
+func TestSubscriberCountFollowsAddAndRemove(t *testing.T) {
+ hub := newTestHub(t, storeWithMail())
+ if hub.SubscriberCount() != 0 {
+ t.Fatalf("a new hub has %d subscribers", hub.SubscriberCount())
+ }
+
+ sub := subscribeTo(hub, mailboxOne.String(), 1)
+ if hub.SubscriberCount() != 1 {
+ t.Errorf("after one subscriber: %d", hub.SubscriberCount())
+ }
+
+ hub.remove(sub)
+ if hub.SubscriberCount() != 0 {
+ t.Errorf("after removing it: %d", hub.SubscriberCount())
+ }
+}
+
+// Publishing with nobody listening must be a no-op, not a panic: the relay
+// runs whether or not a browser happens to be open.
+func TestPublishWithNoSubscribersIsHarmless(t *testing.T) {
+ hub := newTestHub(t, storeWithMail())
+ hub.Publish(port.OutboxEvent{
+ EventType: "EmailReceived",
+ AggregateID: uuid.New(),
+ Payload: []byte(`{}`),
+ })
+}
+
+// --- the upgrade's own authentication ------------------------------------
+
+// The session is verified before the upgrade, so an unauthenticated caller
+// gets a plain 401 and never reaches the WebSocket machinery.
+func TestEventStreamRefusesAnUnauthenticatedUpgrade(t *testing.T) {
+ hub := newTestHub(t, storeWithMail())
+
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/events", nil))
+
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("an unauthenticated upgrade answered %d, want 401", rec.Code)
+ }
+}
+
+func TestEventStreamRefusesAnAdminSession(t *testing.T) {
+ hub := newTestHub(t, storeWithMail())
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/events", nil)
+ req.AddCookie(&http.Cookie{Name: "lm_user_session", Value: adminSurfaceToken(t)})
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("an admin session reached the event stream: %d", rec.Code)
+ }
+}
+
+// A valid session for an address this server no longer serves must not open a
+// stream either - that is a deleted account whose token has not expired yet.
+func TestEventStreamRefusesASessionWithNoMailbox(t *testing.T) {
+ store := storeWithMail()
+ delete(store.addresses, sessionOwner)
+ hub := newTestHub(t, store)
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/events", nil)
+ req.AddCookie(&http.Cookie{Name: "lm_user_session", Value: interopSession})
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("a session with no mailbox answered %d, want 401", rec.Code)
+ }
+}
+
+// The payload has to reach the browser unchanged: the page reads the UID out
+// of it to decide what to refresh.
+func TestEventCarriesItsPayloadThrough(t *testing.T) {
+ hub := newTestHub(t, storeWithMail())
+ sub := subscribeTo(hub, mailboxOne.String(), 2)
+
+ hub.Publish(port.OutboxEvent{
+ EventType: "EmailReceived",
+ AggregateID: mailboxOne,
+ Payload: []byte(`{"recipient":"user@example.test","uid":42}`),
+ })
+
+ msg := <-sub.events
+ var payload struct {
+ Recipient string `json:"recipient"`
+ UID int `json:"uid"`
+ }
+ if err := json.Unmarshal(msg.Payload, &payload); err != nil {
+ t.Fatalf("payload did not survive: %v", err)
+ }
+ if payload.UID != 42 || payload.Recipient != "user@example.test" {
+ t.Errorf("payload arrived as %+v", payload)
+ }
+}
diff --git a/services/protocols/internal/presentation/http/mail_api_test.go b/services/protocols/internal/presentation/http/mail_api_test.go
new file mode 100644
index 0000000..2f045f5
--- /dev/null
+++ b/services/protocols/internal/presentation/http/mail_api_test.go
@@ -0,0 +1,539 @@
+package httppresentation
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+
+ "lambdamail/protocols/internal/application/port"
+ "lambdamail/protocols/internal/application/usecase"
+ "lambdamail/protocols/internal/infrastructure/postgres"
+)
+
+// The webmail's HTTP surface: the only part of this server a browser talks to
+// directly, and the one that decides whose mail a session may touch.
+//
+// None of it was covered. Every one of these asserts something a mistake here
+// would cost: another account's mail, an attachment rendered as a page on this
+// origin, or a sender address chosen by the request rather than the session.
+
+// --- test doubles --------------------------------------------------------
+
+// fakeMailStore records what the API asked for, so a handler can be checked
+// on the arguments it passed rather than only on its status code.
+type fakeMailStore struct {
+ // byMailbox is the mail each mailbox owns, keyed by mailbox ID.
+ byMailbox map[string][]port.WebmailMessage
+ addresses map[string]string // address -> mailbox ID
+
+ seenCalls []string
+ deletedCalls []string
+ movedCalls []string
+ expungeErr error
+ moveErr error
+ blob uuid.UUID
+}
+
+func (f *fakeMailStore) FindMailboxIDByAddress(_ context.Context, address string) (string, error) {
+ id, ok := f.addresses[address]
+ if !ok {
+ return "", postgres.ErrMessageNotFound
+ }
+ return id, nil
+}
+
+func (f *fakeMailStore) ListFolders(_ context.Context, mailboxID string) ([]port.WebmailFolder, error) {
+ return []port.WebmailFolder{{
+ ID: "f1", Name: "INBOX", SpecialUse: "inbox",
+ TotalCount: len(f.byMailbox[mailboxID]),
+ }}, nil
+}
+
+func (f *fakeMailStore) ListMessages(
+ _ context.Context, mailboxID, _, _ string, _, _ int,
+) ([]port.WebmailMessage, error) {
+ return f.byMailbox[mailboxID], nil
+}
+
+func (f *fakeMailStore) GetMessageBlob(
+ _ context.Context, mailboxID, _ string, uid uint32,
+) (uuid.UUID, error) {
+ for _, m := range f.byMailbox[mailboxID] {
+ if m.UID == uid {
+ return f.blob, nil
+ }
+ }
+ // Exactly what the real repository does for a UID this mailbox does not
+ // own: not found, indistinguishable from one that does not exist.
+ return uuid.Nil, postgres.ErrMessageNotFound
+}
+
+func (f *fakeMailStore) MarkSeen(_ context.Context, mailboxID, folder string, uid uint32, seen bool) error {
+ f.seenCalls = append(f.seenCalls, mailboxID+"/"+folder)
+ return nil
+}
+
+func (f *fakeMailStore) Expunge(_ context.Context, mailboxID, folder string, uid uint32) error {
+ if f.expungeErr != nil {
+ return f.expungeErr
+ }
+ f.expungeErr = nil
+ f.deletedCalls = append(f.deletedCalls, mailboxID+"/"+folder)
+ return nil
+}
+
+func (f *fakeMailStore) MoveToTrash(
+ _ context.Context, mailboxID, folder string, uid uint32,
+) (uint32, error) {
+ if f.moveErr != nil {
+ return 0, f.moveErr
+ }
+ f.movedCalls = append(f.movedCalls, mailboxID+"/"+folder)
+ return uid, nil
+}
+
+type fakeBlobReader struct{ raw []byte }
+
+func (f *fakeBlobReader) ReadByID(context.Context, uuid.UUID) ([]byte, error) {
+ return f.raw, nil
+}
+
+// --- harness -------------------------------------------------------------
+
+// newMailAPI builds the API over fakes, pinned to an instant inside the
+// captured tokens' validity window so these tests never age out.
+func newMailAPI(t *testing.T, store *fakeMailStore, raw []byte) *mailAPI {
+ t.Helper()
+ uc := usecase.NewWebmailUseCase(store, &fakeBlobReader{raw: raw}, nil, nil, "mail.example.test")
+ return &mailAPI{useCase: uc, sessions: testVerifier(interopSecret)}
+}
+
+// The session in interopSession belongs to this address.
+const sessionOwner = "user@example.test"
+
+func request(method, target string, body string) *http.Request {
+ req := httptest.NewRequest(method, target, strings.NewReader(body))
+ req.Header.Set("Authorization", "Bearer "+interopSession)
+ req.Header.Set("Content-Type", "application/json")
+ return req
+}
+
+func storeWithMail() *fakeMailStore {
+ return &fakeMailStore{
+ addresses: map[string]string{sessionOwner: "mailbox-1", "other@example.test": "mailbox-2"},
+ byMailbox: map[string][]port.WebmailMessage{
+ "mailbox-1": {{UID: 1, Subject: "mine"}},
+ "mailbox-2": {{UID: 99, Subject: "somebody else's"}},
+ },
+ blob: uuid.New(),
+ }
+}
+
+// --- authentication ------------------------------------------------------
+
+func TestMailApiRefusesARequestWithNoSession(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ for name, req := range map[string]*http.Request{
+ "folders": httptest.NewRequest(http.MethodGet, "/api/v1/mail/folders", nil),
+ "messages": httptest.NewRequest(http.MethodGet, "/api/v1/mail/messages", nil),
+ "seen": httptest.NewRequest(http.MethodPost, "/api/v1/mail/seen", strings.NewReader(`{"uid":1}`)),
+ "delete": httptest.NewRequest(http.MethodPost, "/api/v1/mail/delete", strings.NewReader(`{"uid":1}`)),
+ } {
+ rec := httptest.NewRecorder()
+ switch name {
+ case "folders":
+ api.handleFolders(rec, req)
+ case "messages":
+ api.handleMessages(rec, req)
+ case "seen":
+ api.handleSeen(rec, req)
+ case "delete":
+ api.handleDelete(rec, req)
+ }
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("%s answered %d without a session, want 401", name, rec.Code)
+ }
+ }
+}
+
+// An admin session must not reach the mail API. The two surfaces are separated
+// by audience precisely so a console token cannot read a mailbox.
+func TestMailApiRefusesAnAdminSession(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/mail/folders", nil)
+ req.Header.Set("Authorization", "Bearer "+adminSurfaceToken(t))
+ rec := httptest.NewRecorder()
+ api.handleFolders(rec, req)
+
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("an admin token reached the mail API: %d", rec.Code)
+ }
+}
+
+// The half-issued token from between the password and the second factor is not
+// a session and must not be accepted as one.
+func TestMailApiRefusesAnMfaChallengeToken(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/mail/folders", nil)
+ req.Header.Set("Authorization", "Bearer "+interopChallenge)
+ rec := httptest.NewRecorder()
+ api.handleFolders(rec, req)
+
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("an MFA challenge token was accepted as a session: %d", rec.Code)
+ }
+}
+
+// A tampered signature must fail, or the secret is doing nothing.
+func TestMailApiRefusesATamperedToken(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/mail/folders", nil)
+ req.Header.Set("Authorization", "Bearer "+interopSession[:len(interopSession)-4]+"AAAA")
+ rec := httptest.NewRecorder()
+ api.handleFolders(rec, req)
+
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("a token with a broken signature was accepted: %d", rec.Code)
+ }
+}
+
+// The cookie is the browser's normal path and must work like the header.
+func TestMailApiAcceptsTheSessionCookie(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/mail/folders", nil)
+ req.AddCookie(&http.Cookie{Name: "lm_user_session", Value: interopSession})
+ rec := httptest.NewRecorder()
+ api.handleFolders(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Errorf("the session cookie was not accepted: %d", rec.Code)
+ }
+}
+
+// --- isolation between accounts -----------------------------------------
+
+// The mailbox is resolved from the session's address, never from the request.
+// This is the check that keeps one account out of another's mail.
+func TestMailApiServesOnlyTheSessionsOwnMailbox(t *testing.T) {
+ store := storeWithMail()
+ api := newMailAPI(t, store, nil)
+
+ rec := httptest.NewRecorder()
+ api.handleMessages(rec, request(http.MethodGet, "/api/v1/mail/messages", ""))
+
+ var got []port.WebmailMessage
+ if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ for _, m := range got {
+ if m.Subject == "somebody else's" {
+ t.Fatal("the API served another mailbox's mail")
+ }
+ }
+ if len(got) != 1 || got[0].Subject != "mine" {
+ t.Errorf("got %+v, want only this session's own message", got)
+ }
+}
+
+// Naming another mailbox's UID must read as not-found, not as that message.
+func TestMailApiCannotReadAnotherMailboxesMessageByUid(t *testing.T) {
+ store := storeWithMail()
+ api := newMailAPI(t, store, []byte("Subject: secret\r\n\r\nbody"))
+
+ rec := httptest.NewRecorder()
+ // UID 99 exists, but in mailbox-2.
+ api.handleMessage(rec, request(http.MethodGet, "/api/v1/mail/message/99", ""))
+
+ if rec.Code != http.StatusNotFound {
+ t.Errorf("reading another mailbox's UID answered %d, want 404", rec.Code)
+ }
+ if strings.Contains(rec.Body.String(), "secret") {
+ t.Error("another mailbox's message body was served")
+ }
+}
+
+// --- reading -------------------------------------------------------------
+
+// Opening a message marks it read; that is what moves the unread counter.
+func TestOpeningAMessageMarksItRead(t *testing.T) {
+ store := storeWithMail()
+ api := newMailAPI(t, store, []byte("Subject: hi\r\n\r\nbody"))
+
+ rec := httptest.NewRecorder()
+ api.handleMessage(rec, request(http.MethodGet, "/api/v1/mail/message/1?folder=inbox", ""))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("reading own message answered %d", rec.Code)
+ }
+ if len(store.seenCalls) != 1 || store.seenCalls[0] != "mailbox-1/inbox" {
+ t.Errorf("marked seen on %v, want exactly [mailbox-1/inbox]", store.seenCalls)
+ }
+}
+
+// The folder defaults rather than 400-ing, because the list view omits it.
+func TestMessageListDefaultsToTheInbox(t *testing.T) {
+ store := storeWithMail()
+ api := newMailAPI(t, store, nil)
+
+ rec := httptest.NewRecorder()
+ api.handleSeen(rec, request(http.MethodPost, "/api/v1/mail/seen", `{"uid":1,"seen":true}`))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("answered %d", rec.Code)
+ }
+ if len(store.seenCalls) != 1 || store.seenCalls[0] != "mailbox-1/inbox" {
+ t.Errorf("the folder did not default to inbox: %v", store.seenCalls)
+ }
+}
+
+// --- deleting ------------------------------------------------------------
+
+func TestDeleteMovesToTrashFromAnOrdinaryFolder(t *testing.T) {
+ store := storeWithMail()
+ api := newMailAPI(t, store, nil)
+
+ rec := httptest.NewRecorder()
+ api.handleDelete(rec, request(http.MethodPost, "/api/v1/mail/delete", `{"folder":"inbox","uid":1}`))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("delete answered %d: %s", rec.Code, rec.Body)
+ }
+ // The mailbox is asserted, not just the call: counting calls alone would
+ // pass just as happily if the delete acted on somebody else's mailbox.
+ if len(store.movedCalls) != 1 || store.movedCalls[0] != "mailbox-1/inbox" {
+ t.Errorf("moved %v, want exactly [mailbox-1/inbox]", store.movedCalls)
+ }
+ if len(store.deletedCalls) != 0 {
+ t.Errorf("deleting from the inbox destroyed the message: %v", store.deletedCalls)
+ }
+}
+
+func TestDeleteFromTrashExpungesForGood(t *testing.T) {
+ store := storeWithMail()
+ store.moveErr = port.ErrAlreadyInTrash
+ api := newMailAPI(t, store, nil)
+
+ rec := httptest.NewRecorder()
+ api.handleDelete(rec, request(http.MethodPost, "/api/v1/mail/delete", `{"folder":"trash","uid":1}`))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("delete from trash answered %d: %s", rec.Code, rec.Body)
+ }
+ if len(store.deletedCalls) != 1 || store.deletedCalls[0] != "mailbox-1/trash" {
+ t.Errorf("expunged %v, want exactly [mailbox-1/trash]", store.deletedCalls)
+ }
+}
+
+// A delete with no UID is a bug in the caller, not a request to delete UID 0.
+func TestDeleteRejectsAMissingUid(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ rec := httptest.NewRecorder()
+ api.handleDelete(rec, request(http.MethodPost, "/api/v1/mail/delete", `{"folder":"inbox"}`))
+
+ if rec.Code != http.StatusBadRequest {
+ t.Errorf("a delete with no UID answered %d, want 400", rec.Code)
+ }
+}
+
+func TestDeleteRejectsMalformedJson(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ rec := httptest.NewRecorder()
+ api.handleDelete(rec, request(http.MethodPost, "/api/v1/mail/delete", `{not json`))
+
+ if rec.Code != http.StatusBadRequest {
+ t.Errorf("malformed JSON answered %d, want 400", rec.Code)
+ }
+}
+
+// --- error mapping -------------------------------------------------------
+
+// Each domain error has to reach the browser as something the composer can act
+// on. A send limit that arrives as a 500 tells the user to try again, which is
+// the one thing that will not work.
+func TestDomainErrorsMapToUsefulStatusCodes(t *testing.T) {
+ cases := []struct {
+ err error
+ want int
+ }{
+ {postgres.ErrMessageNotFound, http.StatusNotFound},
+ {usecase.ErrNoSuchMailbox, http.StatusNotFound},
+ {usecase.ErrNoRecipients, http.StatusBadRequest},
+ {usecase.ErrSendLimitExceeded, http.StatusTooManyRequests},
+ {usecase.ErrSenderNotOwned, http.StatusForbidden},
+ {usecase.ErrDraftsUnavailable, http.StatusServiceUnavailable},
+ {errors.New("something unexpected"), http.StatusInternalServerError},
+ }
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ for _, tc := range cases {
+ rec := httptest.NewRecorder()
+ api.fail(rec, tc.err)
+ if rec.Code != tc.want {
+ t.Errorf("%v mapped to %d, want %d", tc.err, rec.Code, tc.want)
+ }
+ // The internals must not travel to the browser with it.
+ if strings.Contains(rec.Body.String(), "something unexpected") {
+ t.Errorf("an internal error message leaked to the client: %s", rec.Body)
+ }
+ }
+}
+
+// adminSurfaceToken mints a token for the admin surface under the same secret,
+// to prove the audience check is what refuses it rather than the signature.
+func adminSurfaceToken(t *testing.T) string {
+ t.Helper()
+ return mintSession(t, WebSession{
+ Subject: "mb-1", Email: sessionOwner, Role: "SUPER_ADMIN",
+ Surface: "admin", Audience: "lambdamail:admin",
+ Purpose: "session", MfaSatisfied: true,
+ IssuedAt: interopIssuedAt,
+ ExpiresAt: interopIssuedAt + 3600,
+ })
+}
+
+var _ = time.Now
+
+// --- composing -----------------------------------------------------------
+
+// The sender is taken from the session and never from the request body.
+// Letting the body choose From is how a webmail becomes an open relay for
+// spoofed mail, signed with this domain's own DKIM key.
+func TestComposeAlwaysSendsAsTheSessionsOwnAddress(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ rec := httptest.NewRecorder()
+ req := request(http.MethodPost, "/api/v1/mail/send", `{
+ "from":"ceo@bank.example",
+ "to":["victim@example.test"],
+ "subject":"Wire transfer",
+ "html":"urgent
"
+ }`)
+
+ input, ok := api.decodeCompose(rec, req, sessionOwner)
+ if !ok {
+ t.Fatalf("decode refused a valid body: %s", rec.Body)
+ }
+ if input.From != sessionOwner {
+ t.Errorf("From is %q; the request body chose the sender", input.From)
+ }
+}
+
+// Bcc has to survive decoding, or a blind copy is silently dropped and nobody
+// is told the message did not reach them.
+func TestComposeKeepsBlindCopies(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ rec := httptest.NewRecorder()
+ input, ok := api.decodeCompose(rec, request(http.MethodPost, "/api/v1/mail/send", `{
+ "to":["a@example.test"],"cc":["b@example.test"],"bcc":["c@example.test"],"text":"hi"
+ }`), sessionOwner)
+ if !ok {
+ t.Fatalf("decode refused: %s", rec.Body)
+ }
+ if len(input.Bcc) != 1 || input.Bcc[0] != "c@example.test" {
+ t.Errorf("Bcc arrived as %v", input.Bcc)
+ }
+ if len(input.Cc) != 1 {
+ t.Errorf("Cc arrived as %v", input.Cc)
+ }
+}
+
+// The composer sends "html"; the field was once read as "body", and every
+// message went out blank. The legacy name still has to work for anything
+// written against the old shape.
+func TestComposeAcceptsBothTheCurrentAndLegacyBodyFields(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ rec := httptest.NewRecorder()
+ current, ok := api.decodeCompose(rec, request(http.MethodPost, "/x",
+ `{"to":["a@example.test"],"html":"real
"}`), sessionOwner)
+ if !ok || current.HTML != "real
" {
+ t.Errorf("the html field did not survive: %+v", current)
+ }
+
+ rec = httptest.NewRecorder()
+ legacy, ok := api.decodeCompose(rec, request(http.MethodPost, "/x",
+ `{"to":["a@example.test"],"body":"plain words"}`), sessionOwner)
+ if !ok || legacy.Text != "plain words" {
+ t.Errorf("the legacy body field was dropped: %+v", legacy)
+ }
+}
+
+// The draft a sent message came from has to reach the use case, or the draft
+// outlives the message it became.
+func TestComposeCarriesTheDraftItCameFrom(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ rec := httptest.NewRecorder()
+ input, ok := api.decodeCompose(rec, request(http.MethodPost, "/api/v1/mail/send",
+ `{"to":["a@example.test"],"text":"hi","draft_uid":77}`), sessionOwner)
+ if !ok {
+ t.Fatalf("decode refused: %s", rec.Body)
+ }
+ if input.DraftUID != 77 {
+ t.Errorf("DraftUID is %d, want 77", input.DraftUID)
+ }
+}
+
+// The draft decode reads the body once and has to get both halves out of it.
+func TestDraftDecodeReadsBothTheMessageAndTheUidItReplaces(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ rec := httptest.NewRecorder()
+ input, replaceUID, ok := api.decodeDraft(rec, request(http.MethodPost, "/api/v1/mail/draft",
+ `{"to":["a@example.test"],"subject":"half","html":"writ
","replace_uid":12}`), sessionOwner)
+ if !ok {
+ t.Fatalf("decode refused: %s", rec.Body)
+ }
+ if replaceUID != 12 {
+ t.Errorf("replace_uid is %d, want 12", replaceUID)
+ }
+ if input.Subject != "half" || input.HTML != "writ
" {
+ t.Errorf("the draft body was lost: %+v", input)
+ }
+ if input.From != sessionOwner {
+ t.Errorf("a draft's From is %q, want the session's address", input.From)
+ }
+}
+
+// A draft with no replace_uid is the first save of a message and must still
+// decode - it simply supersedes nothing.
+func TestDraftDecodeToleratesAMissingReplaceUid(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ rec := httptest.NewRecorder()
+ _, replaceUID, ok := api.decodeDraft(rec, request(http.MethodPost, "/api/v1/mail/draft",
+ `{"subject":"first save"}`), sessionOwner)
+ if !ok {
+ t.Fatalf("the first save of a draft was refused: %s", rec.Body)
+ }
+ if replaceUID != 0 {
+ t.Errorf("replace_uid is %d, want 0", replaceUID)
+ }
+}
+
+func TestComposeRejectsMalformedJson(t *testing.T) {
+ api := newMailAPI(t, storeWithMail(), nil)
+
+ rec := httptest.NewRecorder()
+ if _, ok := api.decodeCompose(rec, request(http.MethodPost, "/x", `{"to":`), sessionOwner); ok {
+ t.Error("malformed JSON was accepted")
+ }
+ if rec.Code != http.StatusBadRequest {
+ t.Errorf("answered %d, want 400", rec.Code)
+ }
+}
diff --git a/services/protocols/internal/presentation/http/websession_test.go b/services/protocols/internal/presentation/http/websession_test.go
index 4848eb5..eed9db1 100644
--- a/services/protocols/internal/presentation/http/websession_test.go
+++ b/services/protocols/internal/presentation/http/websession_test.go
@@ -1,6 +1,10 @@
package httppresentation
import (
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/json"
"errors"
"testing"
"time"
@@ -91,3 +95,44 @@ func TestWebSessionVerifier_RejectsExpiredToken(t *testing.T) {
t.Fatalf("error = %v, want ErrTokenExpired", err)
}
}
+
+// mintSession signs a session token the way the auth service does, so a test
+// can build a claim set the captured tokens do not cover - another surface, an
+// expired window, a different address.
+//
+// The captured tokens above stay the interop check: this only produces the
+// variations, and produces them through the same algorithm.
+func mintSession(t *testing.T, claims WebSession) string {
+ t.Helper()
+ header := base64.RawURLEncoding.EncodeToString(
+ []byte(`{"alg":"HS256","typ":"JWT"}`))
+ body, err := json.Marshal(claims)
+ if err != nil {
+ t.Fatalf("marshal claims: %v", err)
+ }
+ payload := base64.RawURLEncoding.EncodeToString(body)
+
+ mac := hmac.New(sha256.New, []byte(interopSecret))
+ mac.Write([]byte(header + "." + payload))
+ signature := base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
+
+ return header + "." + payload + "." + signature
+}
+
+// The minted tokens have to verify, or every test built on them is asserting
+// against a signature this server would reject for the wrong reason.
+func TestMintedSessionVerifiesLikeARealOne(t *testing.T) {
+ token := mintSession(t, WebSession{
+ Subject: "mb-1", Email: "user@example.test", Surface: "user",
+ Audience: "lambdamail:user", Purpose: "session", MfaSatisfied: true,
+ IssuedAt: interopIssuedAt, ExpiresAt: interopIssuedAt + 3600,
+ })
+
+ session, err := testVerifier(interopSecret).RequireSurface(token, "user")
+ if err != nil {
+ t.Fatalf("a locally minted session was rejected: %v", err)
+ }
+ if session.Email != "user@example.test" {
+ t.Errorf("Email = %q", session.Email)
+ }
+}
diff --git a/services/webmail/messages/en.json b/services/webmail/messages/en.json
index 0a4b87f..4a0a880 100644
--- a/services/webmail/messages/en.json
+++ b/services/webmail/messages/en.json
@@ -172,6 +172,7 @@
"insertUrl": "Enter the URL:",
"italic": "Italic",
"junk": "Spam",
+ "reports": "Reports",
"loadImages": "Load images",
"markRead": "Mark as read",
"markUnread": "Mark as unread",
diff --git a/services/webmail/messages/es.json b/services/webmail/messages/es.json
index 14b2071..d4ba9e1 100644
--- a/services/webmail/messages/es.json
+++ b/services/webmail/messages/es.json
@@ -172,6 +172,7 @@
"insertUrl": "Introduzca la URL:",
"italic": "Cursiva",
"junk": "Spam",
+ "reports": "Informes",
"loadImages": "Cargar imágenes",
"markRead": "Marcar como leído",
"markUnread": "Marcar como no leído",
diff --git a/services/webmail/messages/pt-BR.json b/services/webmail/messages/pt-BR.json
index a73ca10..12b6463 100644
--- a/services/webmail/messages/pt-BR.json
+++ b/services/webmail/messages/pt-BR.json
@@ -172,6 +172,7 @@
"insertUrl": "Digite a URL:",
"italic": "Itálico",
"junk": "Spam",
+ "reports": "Relatórios",
"loadImages": "Carregar imagens",
"markRead": "Marcar como lida",
"markUnread": "Marcar como não lida",
diff --git a/services/webmail/src/app/(user)/user/layout.tsx b/services/webmail/src/app/(user)/user/layout.tsx
index 498e2d6..2c301e5 100644
--- a/services/webmail/src/app/(user)/user/layout.tsx
+++ b/services/webmail/src/app/(user)/user/layout.tsx
@@ -43,6 +43,9 @@ export default function UserWebmailLayout({ children }: { children: React.ReactN
{ href: "/user/mail/archive", label: t("mail.archive"), icon: Archive, role: "archive" },
{ href: "/user/mail/junk", label: t("mail.junk"), icon: AlertTriangle, role: "junk" },
{ href: "/user/mail/trash", label: t("mail.trash"), icon: Trash2, role: "trash" },
+ // Where delivered DMARC and TLS-RPT reports are filed. Matched by name
+ // rather than a special-use role, because IMAP defines none for reports.
+ { href: "/user/mail/reports", label: t("mail.reports"), icon: ShieldCheck, role: "reports" },
];
const handleLogout = () => {
diff --git a/services/webmail/src/app/(user)/user/mail/[folder]/page.tsx b/services/webmail/src/app/(user)/user/mail/[folder]/page.tsx
index 7c4bd9f..556c33d 100644
--- a/services/webmail/src/app/(user)/user/mail/[folder]/page.tsx
+++ b/services/webmail/src/app/(user)/user/mail/[folder]/page.tsx
@@ -306,6 +306,7 @@ export default function MailFolderPage({ params }: { params: Promise<{ folder: s
archive: t("mail.archive"),
junk: t("mail.junk"),
trash: t("mail.trash"),
+ reports: t("mail.reports"),
};
const folderTitle = FOLDER_LABELS[folder.toLowerCase()] ?? folder;
diff --git a/services/webmail/src/compose-draft-integrity.test.ts b/services/webmail/src/compose-draft-integrity.test.ts
index 963c580..67788ef 100644
--- a/services/webmail/src/compose-draft-integrity.test.ts
+++ b/services/webmail/src/compose-draft-integrity.test.ts
@@ -80,3 +80,23 @@ describe("deleting a message", () => {
expect(del.slice(0, 1200)).toMatch(/setMessages\(/);
});
});
+
+describe("the Reports folder", () => {
+ // DMARC and TLS-RPT reports are parsed on arrival and filed here instead of
+ // the inbox. The sidebar is a fixed list, so without an entry the folder
+ // exists and receives mail that no screen can reach.
+ it("is reachable from the sidebar", () => {
+ const layout = code("src/app/(user)/user/layout.tsx");
+ expect(layout).toContain("/user/mail/reports");
+ });
+
+ it("is named through the translator, not hardcoded", () => {
+ const layout = code("src/app/(user)/user/layout.tsx");
+ expect(layout).toMatch(/t\("mail\.reports"\)/);
+ });
+
+ it("has a title on the message list too", () => {
+ const page = code(MAIL_PAGE);
+ expect(page).toMatch(/reports:\s*t\("mail\.reports"\)/);
+ });
+});