Add end-to-end tests for TLS RPT report ingestion and WebSocket event handling

- Implemented tests for ingesting TLS RPT reports from emails and ensuring they are stored correctly in the database.
- Added tests to verify that the same report can be ingested multiple times without duplication.
- Created tests for backfilling reports that were previously delivered to the inbox.
- Developed WebSocket event handling tests to ensure events are correctly routed to subscribers based on mailbox ownership.
- Included tests for the mail API to validate session handling, mailbox isolation, and error mapping.
- Ensured that the API rejects unauthorized requests and handles malformed JSON gracefully.
This commit is contained in:
Lucas Rayan committed 2026-08-21 09:55:37 -03:00
1 parent 880dced9e4
commit f330c4324e
28 files changed
+2438 -6

No files matched your search

@@ -0,0 +1,50 @@
package main
import (
"context"
"log"
"time"
"lambdamail/protocols/internal/application/usecase"
"lambdamail/protocols/internal/infrastructure/diskstorage"
"lambdamail/protocols/internal/infrastructure/postgres"
)
// runBackfillReports parses the DMARC and TLS-RPT reports that were delivered
// before automatic ingestion existed.
//
// Run by hand, once, after deploying ingestion: every report received until
// then was filed in the admin's inbox as an attachment nothing ever read, so
// the tables the console shows start empty however long the server has been
// running. Safe to run more than once - storing a report is idempotent, and a
// message already filed in Reports is not picked up again.
func runBackfillReports(cfg config) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL)
if err != nil {
log.Fatalf("backfill: cannot reach the database: %v", err)
}
defer pool.Close()
// The reader alone: the backfill only ever reads stored messages back out.
blobs := diskstorage.NewLocalDiskBlobReader(pool)
store := postgres.NewDeliveredReportStore(pool)
uc := usecase.NewBackfillReportsUseCase(store, blobs,
usecase.NewIngestDeliveredReportsUseCase(
usecase.NewIngestReportsUseCase(postgres.NewReportRepository(pool))))
summary, err := uc.Run(ctx, 0)
if err != nil {
log.Fatalf("backfill: %v", err)
}
log.Printf("backfill: scanned %d stored reports, ingested %d, could not read %d",
summary.Scanned, summary.Ingested, summary.Failed)
if summary.Failed > 0 {
log.Printf("backfill: the unreadable ones are filed in Reports with their " +
"bytes intact, so this can be run again once the cause is fixed")
}
}
@@ -59,8 +59,11 @@ func main() {
case "healthcheck":
runHealthcheck()
return
case "backfill-reports":
runBackfillReports(cfg)
return
default:
log.Fatalf("unknown subcommand %q (expected preflight or healthcheck)", os.Args[1])
log.Fatalf("unknown subcommand %q (expected preflight, healthcheck or backfill-reports)", os.Args[1])
}
}
@@ -118,6 +121,16 @@ func run(cfg config) {
inboundUC.SetScanner(scanner)
}
// DMARC and TLS-RPT reports arrive as ordinary mail to the dmarc@ and
// tlsrpt@ aliases every domain gets. Reading them here is what fills the
// report tables the admin console shows; without this they arrived as
// attachments in the admin's inbox and nothing ever parsed one.
//
// It shares the report use case with the HTTP ingest endpoints below, so
// mail and API writes land in the same tables through the same parsers.
reportsUC := usecase.NewIngestReportsUseCase(reportRepo)
inboundUC.SetReportIngestor(usecase.NewIngestDeliveredReportsUseCase(reportsUC))
// ------------------------------------------------------------ outbound
var signer port.DkimSigner
if dkimRepo != nil {
@@ -173,7 +186,7 @@ func run(cfg config) {
postgres.NewWebmailRepository(pool), blobReader, submissionUC, authRepo, cfg.PrimaryMailHost,
).WithLocalFiling(blobs, messages)
router := httppresentation.NewRouter(usecase.NewIngestReportsUseCase(reportRepo), func() error { return pool.Ping(ctx) })
router := httppresentation.NewRouter(reportsUC, func() error { return pool.Ping(ctx) })
if cfg.JwtSecret == "" {
log.Printf("JWT_SECRET is not set: the webmail mail API stays disabled")
}
@@ -3,6 +3,8 @@ package port
import (
"context"
"github.com/google/uuid"
"lambdamail/protocols/internal/domain/entity"
)
@@ -11,3 +13,21 @@ type ReportRepository interface {
SaveDmarcReport(ctx context.Context, report *entity.DmarcReport) error
SaveTlsRptReport(ctx context.Context, report *entity.TlsRptReport) error
}
// StoredReportMessage is one already-delivered message that looks like a
// report: it was addressed to a report alias and is still filed in a mailbox.
type StoredReportMessage struct {
MessageID uuid.UUID
BlobID uuid.UUID
RecipientAddress string
}
// DeliveredReportStore reaches the reports that were delivered before
// automatic ingestion existed, so they can be parsed after the fact.
type DeliveredReportStore interface {
// ListUningestedReports returns stored messages addressed to a report
// alias. limit caps the batch; 0 means a full pass.
ListUningestedReports(ctx context.Context, limit int) ([]StoredReportMessage, error)
// MoveToReportsFolder files a backfilled message out of the inbox.
MoveToReportsFolder(ctx context.Context, messageID uuid.UUID) error
}
@@ -0,0 +1,89 @@
package usecase
import (
"context"
"fmt"
"log"
"lambdamail/protocols/internal/application/port"
)
// BackfillReportsUseCase parses reports that were delivered before automatic
// ingestion existed.
//
// Every report received until now was filed in the admin's inbox as an
// attachment and never read, so the tables the admin console shows start
// empty however long the server has been running. The messages are still
// stored, so the backlog can be parsed after the fact and filed out of the
// inbox at the same time.
type BackfillReportsUseCase struct {
store port.DeliveredReportStore
blobs port.BlobReader
ingestor *IngestDeliveredReportsUseCase
}
func NewBackfillReportsUseCase(
store port.DeliveredReportStore,
blobs port.BlobReader,
ingestor *IngestDeliveredReportsUseCase,
) *BackfillReportsUseCase {
return &BackfillReportsUseCase{store: store, blobs: blobs, ingestor: ingestor}
}
// BackfillSummary is what one run did, for the operator running it by hand.
type BackfillSummary struct {
// Scanned is how many stored report messages were examined.
Scanned int
// Ingested is how many produced a report row.
Ingested int
// Failed is how many could not be read or parsed. They are left filed in
// Reports with their bytes intact, so a later run can retry them.
Failed int
}
// Run parses the backlog. limit caps how many messages are examined; 0 means
// whatever the store considers a full pass.
//
// Only a failure to read the backlog itself stops the run. An individual
// message that cannot be read or parsed is counted and skipped: one corrupt
// attachment from years ago must not block every report behind it.
func (uc *BackfillReportsUseCase) Run(ctx context.Context, limit int) (BackfillSummary, error) {
summary := BackfillSummary{}
pending, err := uc.store.ListUningestedReports(ctx, limit)
if err != nil {
return summary, fmt.Errorf("list the report backlog: %w", err)
}
for _, msg := range pending {
summary.Scanned++
raw, err := uc.blobs.ReadByID(ctx, msg.BlobID)
if err != nil {
summary.Failed++
log.Printf("backfill: could not read the message for %s: %v", msg.RecipientAddress, err)
continue
}
outcome, err := uc.ingestor.Ingest(ctx, []string{msg.RecipientAddress}, raw)
switch {
case err != nil:
summary.Failed++
log.Printf("backfill: could not ingest the message for %s: %v", msg.RecipientAddress, err)
case outcome.Stored:
summary.Ingested++
default:
// Recognised as a report but nothing parseable inside it.
summary.Failed++
}
// Filed either way. The inbox is being cleared, and a report this
// server cannot parse today is still a report - its bytes stay on
// disk, so a later run can try again once the parser is fixed.
if err := uc.store.MoveToReportsFolder(ctx, msg.MessageID); err != nil {
log.Printf("backfill: could not file %s into %s: %v", msg.MessageID, ReportsFolderName, err)
}
}
return summary, nil
}
@@ -0,0 +1,172 @@
package usecase
import (
"context"
"errors"
"testing"
"github.com/google/uuid"
"lambdamail/protocols/internal/application/port"
)
// Reports that arrived before ingestion existed are still sitting in the
// admin's inbox. They are ordinary stored messages, so they can be read back
// out and parsed after the fact.
type stubDeliveredReportStore struct {
pending []port.StoredReportMessage
filed []uuid.UUID
blobs map[uuid.UUID][]byte
listErr error
fileErr error
blobErr error
}
func (s *stubDeliveredReportStore) ListUningestedReports(
_ context.Context, limit int,
) ([]port.StoredReportMessage, error) {
if s.listErr != nil {
return nil, s.listErr
}
if limit > 0 && len(s.pending) > limit {
return s.pending[:limit], nil
}
return s.pending, nil
}
func (s *stubDeliveredReportStore) MoveToReportsFolder(_ context.Context, messageID uuid.UUID) error {
if s.fileErr != nil {
return s.fileErr
}
s.filed = append(s.filed, messageID)
return nil
}
func (s *stubDeliveredReportStore) ReadByID(_ context.Context, blobID uuid.UUID) ([]byte, error) {
if s.blobErr != nil {
return nil, s.blobErr
}
body, ok := s.blobs[blobID]
if !ok {
return nil, errors.New("blob not found")
}
return body, nil
}
func newBackfill(store *stubDeliveredReportStore, repo *recordingReportRepo) *BackfillReportsUseCase {
return NewBackfillReportsUseCase(
store, store, NewIngestDeliveredReportsUseCase(NewIngestReportsUseCase(repo)))
}
func TestBackfillIngestsAReportAlreadyInTheMailbox(t *testing.T) {
blobID := uuid.New()
messageID := uuid.New()
payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
store := &stubDeliveredReportStore{
pending: []port.StoredReportMessage{
{MessageID: messageID, BlobID: blobID, RecipientAddress: "tlsrpt@example.test"},
},
blobs: map[uuid.UUID][]byte{blobID: payload},
}
repo := &recordingReportRepo{}
summary, err := newBackfill(store, repo).Run(context.Background(), 0)
if err != nil {
t.Fatalf("run: %v", err)
}
if len(repo.tlsRpt) != 1 {
t.Fatalf("the stored report was not ingested: %d", len(repo.tlsRpt))
}
if repo.tlsRpt[0].Domain != "example.test" {
t.Errorf("ingested domain is %q", repo.tlsRpt[0].Domain)
}
if summary.Ingested != 1 {
t.Errorf("summary reports %d ingested, want 1", summary.Ingested)
}
// Filed out of the inbox, which is the point of running this at all.
if len(store.filed) != 1 || store.filed[0] != messageID {
t.Errorf("the message was not moved to Reports: %v", store.filed)
}
}
// Running it twice must be safe: the second pass has nothing left to do, and
// storage is idempotent anyway.
func TestBackfillOverAnEmptyBacklogDoesNothing(t *testing.T) {
store := &stubDeliveredReportStore{}
repo := &recordingReportRepo{}
summary, err := newBackfill(store, repo).Run(context.Background(), 0)
if err != nil {
t.Fatalf("run: %v", err)
}
if summary.Ingested != 0 || summary.Failed != 0 {
t.Errorf("an empty backlog produced %+v", summary)
}
}
// One unreadable report must not stop the rest of the backlog: a single
// corrupted attachment from years ago would otherwise block every report
// behind it forever.
func TestBackfillContinuesPastAnUnreadableReport(t *testing.T) {
goodBlob, badBlob := uuid.New(), uuid.New()
goodMsg, badMsg := uuid.New(), uuid.New()
store := &stubDeliveredReportStore{
pending: []port.StoredReportMessage{
{MessageID: badMsg, BlobID: badBlob, RecipientAddress: "tlsrpt@example.test"},
{MessageID: goodMsg, BlobID: goodBlob, RecipientAddress: "tlsrpt@example.test"},
},
blobs: map[uuid.UUID][]byte{
badBlob: messageWithAttachment("broken.json.gz", "application/gzip", []byte("rubbish")),
goodBlob: messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON)),
},
}
repo := &recordingReportRepo{}
summary, err := newBackfill(store, repo).Run(context.Background(), 0)
if err != nil {
t.Fatalf("run: %v", err)
}
if summary.Ingested != 1 {
t.Errorf("the readable report was not ingested: %+v", summary)
}
if summary.Failed != 1 {
t.Errorf("the unreadable one was not reported: %+v", summary)
}
// Both are filed either way - the inbox is being cleared, and a report
// this server cannot parse is still a report.
if len(store.filed) != 2 {
t.Errorf("filed %d messages, want both", len(store.filed))
}
}
// A blob that cannot be read at all is counted and skipped, never fatal.
func TestBackfillSurvivesAMissingBlob(t *testing.T) {
store := &stubDeliveredReportStore{
pending: []port.StoredReportMessage{
{MessageID: uuid.New(), BlobID: uuid.New(), RecipientAddress: "dmarc@example.test"},
},
blobs: map[uuid.UUID][]byte{},
}
repo := &recordingReportRepo{}
summary, err := newBackfill(store, repo).Run(context.Background(), 0)
if err != nil {
t.Fatalf("a missing blob must not fail the run: %v", err)
}
if summary.Failed != 1 {
t.Errorf("the missing blob was not counted: %+v", summary)
}
}
// A failure listing the backlog is the one thing worth stopping for: it means
// the database is unreachable, and every message would be counted as failed.
func TestBackfillStopsIfItCannotReadTheBacklog(t *testing.T) {
store := &stubDeliveredReportStore{listErr: errors.New("database is down")}
if _, err := newBackfill(store, &recordingReportRepo{}).Run(context.Background(), 0); err == nil {
t.Error("a backlog that cannot be listed should be reported as an error")
}
}
@@ -0,0 +1,188 @@
package usecase
import (
"bytes"
"context"
"io"
"log"
"strings"
"github.com/emersion/go-message"
)
// ReportKind says what a delivered message is, decided by the address it was
// sent to.
type ReportKind string
const (
// ReportKindNone is ordinary mail, which this use case leaves alone.
ReportKindNone ReportKind = ""
ReportKindDmarc ReportKind = "dmarc"
ReportKindTlsRpt ReportKind = "tlsrpt"
)
// reportLocalParts maps the mailbox a report is addressed to onto what it
// contains.
//
// The address is the only trustworthy signal. The subject and the attachment
// filename are both chosen by the sender, so keying on either would let anyone
// who can send this server a message write rows into the report tables. These
// local parts exist because EnsureSystemAliases creates them for every domain.
var reportLocalParts = map[string]ReportKind{
"dmarc": ReportKindDmarc,
"tlsrpt": ReportKindTlsRpt,
}
// ReportsFolderName is where an ingested report is filed.
//
// Not the inbox: these are machine-to-machine documents that arrive daily from
// every large receiver, and nobody reads the attachment by hand. Not discarded
// either - keeping the original bytes is what makes a parser fix re-appliable
// to reports already received.
const ReportsFolderName = "Reports"
// IngestDeliveredReportsUseCase reads DMARC and TLS-RPT reports out of the
// mail that carries them.
//
// The parsers, the tables and the HTTP ingest endpoints all existed already,
// but nothing connected inbound mail to them: reports were filed in the
// admin's inbox as attachments nobody could read, and the report tables stayed
// empty however many arrived.
type IngestDeliveredReportsUseCase struct {
reports *IngestReportsUseCase
}
func NewIngestDeliveredReportsUseCase(reports *IngestReportsUseCase) *IngestDeliveredReportsUseCase {
return &IngestDeliveredReportsUseCase{reports: reports}
}
// Ingest parses any report the message carries and reports what kind of
// message it was, so the caller can file it somewhere other than the inbox.
//
// It never returns an error for a bad report. A malformed payload is the
// reporter's mistake and a storage failure is this server's, and neither is a
// reason to answer the SMTP transaction with a failure: the reporter would
// retry the same message on a schedule, forever, and no retry would ever help.
// Failures are logged and the message is still delivered, which also leaves
// the original bytes on disk to re-ingest once the cause is fixed.
func (uc *IngestDeliveredReportsUseCase) Ingest(
ctx context.Context, recipients []string, payload []byte,
) (ReportOutcome, error) {
kind := ReportKindNone
for _, address := range recipients {
if found := reportKindForAddress(address); found != ReportKindNone {
kind = found
break
}
}
if kind == ReportKindNone || uc.reports == nil {
return ReportOutcome{}, nil
}
// Every part is tried in turn, so the covering note a reporter puts in
// front of the attachment is simply one that does not parse. Only the
// last failure is kept, and only reported if no part worked at all:
// logging each one would file an error for the human-readable part of
// every report that ingests perfectly well.
var lastErr error
for _, candidate := range reportPayloads(payload) {
var err error
switch kind {
case ReportKindDmarc:
_, err = uc.reports.IngestDmarc(ctx, candidate)
case ReportKindTlsRpt:
_, err = uc.reports.IngestTlsRpt(ctx, candidate)
}
if err != nil {
lastErr = err
continue
}
// One report per message, so there is nothing to gain from reading
// the remaining parts once one has been stored.
return ReportOutcome{Kind: kind, Stored: true}, nil
}
if lastErr != nil {
// Logged rather than returned: see the note above on why a bad report
// must not fail the delivery.
log.Printf("reports: no readable %s report in a message to %v: %v", kind, recipients, lastErr)
}
// Recognised by its address but nothing could be parsed out of it. Still a
// report, so the caller files it with the others rather than leaving it in
// the inbox.
return ReportOutcome{Kind: kind}, nil
}
// ReportOutcome says what a message turned out to be and whether anything was
// actually stored from it.
//
// The two are separate because they drive different decisions: Kind decides
// where the message is filed, and Stored is how a caller tells a report it
// parsed from one it merely recognised - which Ingest deliberately does not
// report as an error.
type ReportOutcome struct {
Kind ReportKind
Stored bool
}
// reportKindForAddress classifies one recipient address by its local part.
func reportKindForAddress(address string) ReportKind {
local, _, found := strings.Cut(address, "@")
if !found {
return ReportKindNone
}
return reportLocalParts[strings.ToLower(strings.TrimSpace(local))]
}
// reportPayloads returns every part of the message that could be a report,
// decoded.
//
// Every attachment is offered rather than the one whose name looks right:
// filenames vary between reporters and the parsers already reject anything
// that is not a report, so matching on the name would only add a way to miss
// one. The whole body is offered last, for the reporters that send the
// document as the message itself rather than as an attachment.
func reportPayloads(payload []byte) [][]byte {
entity, err := message.Read(bytes.NewReader(payload))
if err != nil {
return [][]byte{payload}
}
candidates := collectReportParts(entity, 0)
return append(candidates, payload)
}
// maxReportParts bounds the walk: a report has one attachment, and a message
// with hundreds of nested parts is not one this server needs to read.
const maxReportParts = 16
// maxReportPartBytes bounds each part. Aggregate reports are XML or JSON in
// the tens of kilobytes even before compression.
const maxReportPartBytes = 32 << 20
func collectReportParts(entity *message.Entity, depth int) [][]byte {
// Bounded so a deliberately deep MIME tree cannot drive this into a
// pathological walk.
if depth > 8 {
return nil
}
if multipart := entity.MultipartReader(); multipart != nil {
out := [][]byte{}
for len(out) < maxReportParts {
part, err := multipart.NextPart()
if err != nil {
break
}
out = append(out, collectReportParts(part, depth+1)...)
}
return out
}
// go-message decodes the transfer encoding as the body is read, so what
// comes back here is the attachment's real bytes rather than its base64.
body, err := io.ReadAll(io.LimitReader(entity.Body, maxReportPartBytes))
if err != nil || len(body) == 0 {
return nil
}
return [][]byte{body}
}
@@ -0,0 +1,318 @@
package usecase
import (
"bytes"
"compress/gzip"
"context"
"encoding/base64"
"errors"
"testing"
"github.com/google/uuid"
"lambdamail/protocols/internal/application/port"
"lambdamail/protocols/internal/domain/entity"
)
// Reports arrive as ordinary mail to the addresses the system aliases
// guarantee exist. Nothing read them: they were filed in the admin's inbox as
// attachments and the parsed tables stayed empty.
type recordingReportRepo struct {
dmarc []*entity.DmarcReport
tlsRpt []*entity.TlsRptReport
err error
}
func (r *recordingReportRepo) SaveDmarcReport(_ context.Context, report *entity.DmarcReport) error {
if r.err != nil {
return r.err
}
r.dmarc = append(r.dmarc, report)
return nil
}
func (r *recordingReportRepo) SaveTlsRptReport(_ context.Context, report *entity.TlsRptReport) error {
if r.err != nil {
return r.err
}
r.tlsRpt = append(r.tlsRpt, report)
return nil
}
// gzipped compresses a payload the way a reporter does before attaching it.
func gzipped(t *testing.T, payload string) []byte {
t.Helper()
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
if _, err := gz.Write([]byte(payload)); err != nil {
t.Fatal(err)
}
if err := gz.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
// messageWithAttachment builds the shape a reporter actually sends: a short
// human-readable part plus the report as a base64 attachment.
func messageWithAttachment(filename, contentType string, content []byte) []byte {
encoded := base64.StdEncoding.EncodeToString(content)
var b bytes.Buffer
b.WriteString("From: noreply-dmarc-support@google.com\r\n")
b.WriteString("Subject: Report Domain: example.test\r\n")
b.WriteString("MIME-Version: 1.0\r\n")
b.WriteString("Content-Type: multipart/mixed; boundary=\"sep\"\r\n\r\n")
b.WriteString("--sep\r\n")
b.WriteString("Content-Type: text/plain; charset=utf-8\r\n\r\n")
b.WriteString("This is an aggregate report.\r\n")
b.WriteString("--sep\r\n")
b.WriteString("Content-Type: " + contentType + "; name=\"" + filename + "\"\r\n")
b.WriteString("Content-Disposition: attachment; filename=\"" + filename + "\"\r\n")
b.WriteString("Content-Transfer-Encoding: base64\r\n\r\n")
for len(encoded) > 76 {
b.WriteString(encoded[:76] + "\r\n")
encoded = encoded[76:]
}
b.WriteString(encoded + "\r\n")
b.WriteString("--sep--\r\n")
return b.Bytes()
}
const tlsRptJSON = `{"organization-name":"Google Inc.",` +
`"date-range":{"start-datetime":"2026-08-19T00:00:00Z","end-datetime":"2026-08-19T23:59:59Z"},` +
`"report-id":"2026-08-19T00:00:00Z_example.test",` +
`"policies":[{"policy":{"policy-type":"no-policy-found","policy-domain":"example.test"},` +
`"summary":{"total-successful-session-count":2,"total-failure-session-count":0}}]}`
// The exact filename Google uses, which is what arrives in the inbox today.
const googleTlsRptFilename = "google.com!example.test!1787097600!1787183999!001.json.gz"
func newReportIngestor(repo *recordingReportRepo) *IngestDeliveredReportsUseCase {
return NewIngestDeliveredReportsUseCase(NewIngestReportsUseCase(repo))
}
func TestTlsRptReportAddressedToTlsRptIsIngested(t *testing.T) {
repo := &recordingReportRepo{}
uc := newReportIngestor(repo)
payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
outcome, err := uc.Ingest(context.Background(), []string{"tlsrpt@example.test"}, payload)
if err != nil {
t.Fatalf("ingest: %v", err)
}
if outcome.Kind != ReportKindTlsRpt {
t.Errorf("kind is %q, want tlsrpt", outcome.Kind)
}
if len(repo.tlsRpt) != 1 {
t.Fatalf("want 1 stored report, got %d", len(repo.tlsRpt))
}
if repo.tlsRpt[0].Domain != "example.test" {
t.Errorf("stored domain is %q", repo.tlsRpt[0].Domain)
}
}
func TestDmarcReportAddressedToDmarcIsIngested(t *testing.T) {
repo := &recordingReportRepo{}
uc := newReportIngestor(repo)
payload := messageWithAttachment(
"google.com!example.test!1787097600!1787183999.xml.gz",
"application/gzip", gzipped(t, dmarcAggregateXMLFixture))
outcome, err := uc.Ingest(context.Background(), []string{"dmarc@example.test"}, payload)
if err != nil {
t.Fatalf("ingest: %v", err)
}
if outcome.Kind != ReportKindDmarc {
t.Errorf("kind is %q, want dmarc", outcome.Kind)
}
if len(repo.dmarc) != 1 {
t.Fatalf("want 1 stored report, got %d", len(repo.dmarc))
}
}
// The address decides, not the subject or the filename - both of which the
// sender chooses freely. Ordinary mail must pass through untouched.
func TestOrdinaryMailIsNotTreatedAsAReport(t *testing.T) {
repo := &recordingReportRepo{}
uc := newReportIngestor(repo)
payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
outcome, err := uc.Ingest(context.Background(), []string{"lucas@example.test"}, payload)
if err != nil {
t.Fatalf("ingest: %v", err)
}
if outcome.Kind != ReportKindNone {
t.Errorf("ordinary mail was classified as %q", outcome.Kind)
}
if len(repo.tlsRpt) != 0 || len(repo.dmarc) != 0 {
t.Error("a report was stored from mail addressed to a person")
}
}
// A report this server cannot parse is still a report: it must be recognised,
// so it files with the others, and it must never fail the delivery. Returning
// an error here would have the reporter retry the same broken payload forever.
func TestUnparseableReportIsRecognisedButNeverFailsDelivery(t *testing.T) {
repo := &recordingReportRepo{}
uc := newReportIngestor(repo)
payload := messageWithAttachment("broken.json.gz", "application/gzip", []byte("not a report at all"))
outcome, err := uc.Ingest(context.Background(), []string{"tlsrpt@example.test"}, payload)
if err != nil {
t.Fatalf("a malformed report must not fail the delivery: %v", err)
}
if outcome.Kind != ReportKindTlsRpt {
t.Errorf("kind is %q; the address still says what this message is", outcome.Kind)
}
if len(repo.tlsRpt) != 0 {
t.Error("an unparseable payload was stored")
}
}
// A storage failure is this server's problem, not the reporter's, and must not
// turn into an SMTP error either.
func TestStorageFailureDoesNotFailDelivery(t *testing.T) {
repo := &recordingReportRepo{err: errors.New("database is down")}
uc := newReportIngestor(repo)
payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
if _, err := uc.Ingest(context.Background(), []string{"tlsrpt@example.test"}, payload); err != nil {
t.Errorf("a storage failure must not fail the delivery: %v", err)
}
}
// Reporters vary the local part: dmarc-reports@, tlsrpt@, and the aliases this
// server creates. Matching is on the local part, case-insensitively.
func TestReportAddressMatchingIsCaseInsensitive(t *testing.T) {
for _, address := range []string{"TLSRPT@example.test", "TlsRpt@Example.Test"} {
if got := reportKindForAddress(address); got != ReportKindTlsRpt {
t.Errorf("%s classified as %q, want tlsrpt", address, got)
}
}
if got := reportKindForAddress("DMARC@example.test"); got != ReportKindDmarc {
t.Errorf("DMARC@ classified as %q", got)
}
}
// A report with no attachment at all - some reporters inline small ones - must
// still not fail, and must still be recognised by its address.
func TestReportWithNoAttachmentDoesNotFail(t *testing.T) {
repo := &recordingReportRepo{}
uc := newReportIngestor(repo)
payload := []byte("From: a@b.test\r\nSubject: hi\r\n\r\nno attachment here\r\n")
outcome, err := uc.Ingest(context.Background(), []string{"tlsrpt@example.test"}, payload)
if err != nil {
t.Fatalf("ingest: %v", err)
}
if outcome.Kind != ReportKindTlsRpt {
t.Errorf("kind is %q", outcome.Kind)
}
}
// A DMARC aggregate report, trimmed to the fields the parser reads.
const dmarcAggregateXMLFixture = `<?xml version="1.0" encoding="UTF-8" ?>
<feedback>
<report_metadata>
<org_name>google.com</org_name>
<report_id>987654321</report_id>
<date_range><begin>1787097600</begin><end>1787183999</end></date_range>
</report_metadata>
<policy_published><domain>example.test</domain><p>quarantine</p></policy_published>
<record>
<row>
<source_ip>203.0.113.10</source_ip>
<count>1</count>
<policy_evaluated><disposition>none</disposition><dkim>pass</dkim><spf>pass</spf></policy_evaluated>
</row>
<identifiers><header_from>example.test</header_from></identifiers>
</record>
</feedback>`
// --- the delivery path ---------------------------------------------------
// A report must be ingested and filed out of the way in one delivery, without
// the address ever having to be a real mailbox the user reads.
func TestDeliveredReportIsIngestedAndFiledInReports(t *testing.T) {
repo := &recordingReportRepo{}
messages := &recordingMessageRepository{}
uc := NewProcessInboundEmailUseCase(nil, &capturingBlobStorage{}, messages)
uc.SetReportIngestor(newReportIngestor(repo))
payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
if err := uc.Handle(context.Background(), ProcessInboundEmailInput{
Sender: "smtp-tls-reporting@google.com",
Recipients: []port.MailboxRecord{{ID: uuid.New()}},
RecipientAddresses: []string{"tlsrpt@example.test"},
Body: bytes.NewReader(payload),
}); err != nil {
t.Fatalf("Handle: %v", err)
}
if len(repo.tlsRpt) != 1 {
t.Fatalf("the report was not ingested: %d stored", len(repo.tlsRpt))
}
if len(messages.persisted) != 1 {
t.Fatalf("want the message filed once, got %d", len(messages.persisted))
}
filed := messages.persisted[0]
if filed.TargetFolderName != ReportsFolderName {
t.Errorf("the report was filed in %q, want %q", filed.TargetFolderName, ReportsFolderName)
}
// It is machine-read mail; raising an unread badge for it would recreate
// the inbox noise this exists to remove.
if !filed.AlreadySeen {
t.Error("the filed report was counted as unread mail")
}
}
// Ordinary mail must reach the inbox exactly as before.
func TestOrdinaryMailStillReachesTheInbox(t *testing.T) {
repo := &recordingReportRepo{}
messages := &recordingMessageRepository{}
uc := NewProcessInboundEmailUseCase(nil, &capturingBlobStorage{}, messages)
uc.SetReportIngestor(newReportIngestor(repo))
if err := uc.Handle(context.Background(), ProcessInboundEmailInput{
Sender: "friend@example.test",
Recipients: []port.MailboxRecord{{ID: uuid.New()}},
RecipientAddresses: []string{"lucas@example.test"},
Body: bytes.NewBufferString("Subject: hello\r\n\r\nhi"),
}); err != nil {
t.Fatalf("Handle: %v", err)
}
filed := messages.persisted[0]
if filed.TargetFolderName == ReportsFolderName {
t.Error("ordinary mail was filed as a report")
}
if filed.AlreadySeen {
t.Error("ordinary mail was filed as already read")
}
}
// With no ingestor configured, delivery must behave exactly as it did before
// this existed - reports included.
func TestDeliveryIsUnchangedWithoutAnIngestor(t *testing.T) {
messages := &recordingMessageRepository{}
uc := NewProcessInboundEmailUseCase(nil, &capturingBlobStorage{}, messages)
payload := messageWithAttachment(googleTlsRptFilename, "application/gzip", gzipped(t, tlsRptJSON))
if err := uc.Handle(context.Background(), ProcessInboundEmailInput{
Sender: "smtp-tls-reporting@google.com",
Recipients: []port.MailboxRecord{{ID: uuid.New()}},
RecipientAddresses: []string{"tlsrpt@example.test"},
Body: bytes.NewReader(payload),
}); err != nil {
t.Fatalf("Handle: %v", err)
}
if messages.persisted[0].TargetFolderName == ReportsFolderName {
t.Error("a report was routed with no ingestor configured")
}
}
@@ -65,12 +65,20 @@ type ProcessInboundEmailUseCase struct {
messages port.InboundMessageRepository
folders port.ImapFolderRepository
trackerManager *MailboxTrackerManager
reportIngestor *IngestDeliveredReportsUseCase
}
func NewProcessInboundEmailUseCase(mailboxes port.MailboxRepository, blobs port.BlobStorage, messages port.InboundMessageRepository) *ProcessInboundEmailUseCase {
return &ProcessInboundEmailUseCase{mailboxes: mailboxes, blobs: blobs, messages: messages}
}
// SetReportIngestor enables reading DMARC and TLS-RPT reports out of the mail
// that carries them. Optional: with no ingestor a report is delivered like any
// other message, which is what happened before this existed.
func (uc *ProcessInboundEmailUseCase) SetReportIngestor(ingestor *IngestDeliveredReportsUseCase) {
uc.reportIngestor = ingestor
}
func (uc *ProcessInboundEmailUseCase) SetScanner(scanner port.ContentScanner) {
uc.scanner = scanner
}
@@ -234,6 +242,26 @@ func (uc *ProcessInboundEmailUseCase) Handle(ctx context.Context, input ProcessI
}
}
// Reports are read here, after the scan has had its say and before the
// message is filed, because this is where the recipient list and the
// finished payload are both in hand.
//
// A report that fails to parse is still routed to Reports: the address it
// was sent to is what makes it one, and leaving a broken report in the
// inbox instead would put back exactly the noise this removes. The bytes
// are kept either way, so it can be re-ingested once the cause is fixed.
alreadySeen := false
if uc.reportIngestor != nil {
outcome, err := uc.reportIngestor.Ingest(ctx, input.RecipientAddresses, payload)
if err != nil {
return err
}
if outcome.Kind != ReportKindNone {
targetFolder = ReportsFolderName
alreadySeen = true
}
}
blob, err := uc.blobs.Store(ctx, bytes.NewReader(payload))
if err != nil {
return err
@@ -261,6 +289,7 @@ func (uc *ProcessInboundEmailUseCase) Handle(ctx context.Context, input ProcessI
FromDisplayName: headers.FromDisplayName,
MessageIDHeader: headers.MessageID,
HasAttachments: headers.HasAttachments,
AlreadySeen: alreadySeen,
SPFResult: authResult.SPF,
DKIMResult: authResult.DKIM,
DMARCResult: authResult.DMARC,
@@ -1,6 +1,7 @@
package entity
import (
"archive/zip"
"bytes"
"compress/gzip"
"encoding/xml"
@@ -55,10 +56,16 @@ type dmarcXmlReport struct {
} `xml:"record"`
}
// ParseDmarcXmlReport parses raw XML or gzipped XML DMARC aggregate report.
// ParseDmarcXmlReport parses a DMARC aggregate report: raw XML, gzipped XML,
// or XML inside a zip archive.
//
// Zip matters as much as gzip. Microsoft and Yahoo send .zip, and between them
// they account for a large share of the reports any domain receives; handling
// gzip alone meant those reports never parsed at all.
func ParseDmarcXmlReport(payload []byte) (*DmarcReport, error) {
data := payload
if isGzip(payload) {
switch {
case isGzip(payload):
gz, err := gzip.NewReader(bytes.NewReader(payload))
if err != nil {
return nil, fmt.Errorf("gzip reader: %w", err)
@@ -69,6 +76,12 @@ func ParseDmarcXmlReport(payload []byte) (*DmarcReport, error) {
return nil, fmt.Errorf("decompress gzip: %w", err)
}
data = decompressed
case isZip(payload):
decompressed, err := firstFileInZip(payload)
if err != nil {
return nil, err
}
data = decompressed
}
var parsed dmarcXmlReport
@@ -101,3 +114,41 @@ func ParseDmarcXmlReport(payload []byte) (*DmarcReport, error) {
func isGzip(payload []byte) bool {
return len(payload) >= 2 && payload[0] == 0x1f && payload[1] == 0x8b
}
// isZip reports whether the payload begins with a local file header. The
// empty-archive and spanned-archive signatures are deliberately not accepted:
// neither carries a report.
func isZip(payload []byte) bool {
return len(payload) >= 4 && payload[0] == 'P' && payload[1] == 'K' &&
payload[2] == 0x03 && payload[3] == 0x04
}
// firstFileInZip returns the contents of the first regular file in the
// archive. An aggregate report is one XML document per archive, so there is
// nothing to choose between; directories are skipped so an archive that
// carries one cannot be mistaken for an empty one.
func firstFileInZip(payload []byte) ([]byte, error) {
reader, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
if err != nil {
return nil, fmt.Errorf("zip reader: %w", err)
}
for _, file := range reader.File {
if file.FileInfo().IsDir() {
continue
}
handle, err := file.Open()
if err != nil {
return nil, fmt.Errorf("open %s in zip: %w", file.Name, err)
}
defer handle.Close()
// Bounded so a zip bomb cannot exhaust this process: an aggregate
// report is XML in the tens of kilobytes, and 64 MiB is far past any
// legitimate one.
decompressed, err := io.ReadAll(io.LimitReader(handle, 64<<20))
if err != nil {
return nil, fmt.Errorf("decompress %s in zip: %w", file.Name, err)
}
return decompressed, nil
}
return nil, fmt.Errorf("zip archive contains no report")
}
@@ -0,0 +1,85 @@
package entity
import (
"archive/zip"
"bytes"
"testing"
)
// A real DMARC aggregate report, trimmed to the fields the parser reads.
const dmarcAggregateXML = `<?xml version="1.0" encoding="UTF-8" ?>
<feedback>
<report_metadata>
<org_name>Yahoo</org_name>
<report_id>1234567890</report_id>
<date_range><begin>1787097600</begin><end>1787183999</end></date_range>
</report_metadata>
<policy_published>
<domain>example.test</domain>
<p>quarantine</p>
</policy_published>
<record>
<row>
<source_ip>203.0.113.10</source_ip>
<count>3</count>
<policy_evaluated><disposition>none</disposition><dkim>pass</dkim><spf>pass</spf></policy_evaluated>
</row>
<identifiers><header_from>example.test</header_from></identifiers>
</record>
</feedback>`
// Microsoft and Yahoo send aggregate reports as .zip, not .gz. The parser
// handled gzip only, so every report from those two failed to parse - and
// between them they are a large share of the reports any domain receives.
func TestDmarcReportParsesZippedPayload(t *testing.T) {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
f, err := zw.Create("example.test!1787097600!1787183999.xml")
if err != nil {
t.Fatal(err)
}
if _, err := f.Write([]byte(dmarcAggregateXML)); err != nil {
t.Fatal(err)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
report, err := ParseDmarcXmlReport(buf.Bytes())
if err != nil {
t.Fatalf("parse zipped dmarc report: %v", err)
}
if report.OrgName != "Yahoo" {
t.Errorf("org name is %q, want Yahoo", report.OrgName)
}
if report.Domain != "example.test" {
t.Errorf("domain is %q, want example.test", report.Domain)
}
if len(report.Records) != 1 {
t.Fatalf("want 1 record, got %d", len(report.Records))
}
}
// Plain XML must keep working exactly as before.
func TestDmarcReportStillParsesPlainXml(t *testing.T) {
report, err := ParseDmarcXmlReport([]byte(dmarcAggregateXML))
if err != nil {
t.Fatalf("parse plain xml: %v", err)
}
if report.OrgName != "Yahoo" {
t.Errorf("org name is %q", report.OrgName)
}
}
// An empty archive has no report in it and must be reported as such rather
// than panicking on a missing first file.
func TestDmarcReportRejectsEmptyZip(t *testing.T) {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
if err := zw.Close(); err != nil {
t.Fatal(err)
}
if _, err := ParseDmarcXmlReport(buf.Bytes()); err == nil {
t.Error("an empty zip should not parse as a report")
}
}
@@ -0,0 +1,156 @@
package postgres
import (
"context"
"fmt"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"lambdamail/protocols/internal/application/port"
)
// DeliveredReportStore reaches reports that were delivered before automatic
// ingestion existed and are still filed in a mailbox.
type DeliveredReportStore struct {
pool *pgxpool.Pool
}
func NewDeliveredReportStore(pool *pgxpool.Pool) *DeliveredReportStore {
return &DeliveredReportStore{pool: pool}
}
// reportRecipientPattern matches the local parts the report aliases use. It
// mirrors reportLocalParts in the use case; both exist because this one has to
// be expressible in SQL.
const reportRecipientPattern = `^(dmarc|tlsrpt)@`
// ListUningestedReports returns stored messages addressed to a report alias
// that are not already filed in Reports.
//
// Being outside Reports is what "not yet ingested" means here: the delivery
// path files every report it reads into that folder, so anything addressed to
// a report alias and still sitting elsewhere predates ingestion.
func (s *DeliveredReportStore) ListUningestedReports(
ctx context.Context, limit int,
) ([]port.StoredReportMessage, error) {
if limit <= 0 || limit > 5000 {
limit = 5000
}
rows, err := s.pool.Query(ctx, `
SELECT e.id, e.blob_id, recipient
FROM email_messages e
JOIN folders f ON f.id = e.folder_id
CROSS JOIN LATERAL unnest(e.recipient_addresses) AS recipient
WHERE e.expunged_at IS NULL
AND LOWER(f.name) <> LOWER($1)
AND recipient ~* $2
ORDER BY e.received_at
LIMIT $3
`, reportsFolderName, reportRecipientPattern, limit)
if err != nil {
return nil, fmt.Errorf("list delivered reports: %w", err)
}
defer rows.Close()
out := []port.StoredReportMessage{}
for rows.Next() {
var msg port.StoredReportMessage
if err := rows.Scan(&msg.MessageID, &msg.BlobID, &msg.RecipientAddress); err != nil {
return nil, err
}
out = append(out, msg)
}
return out, rows.Err()
}
// reportsFolderName is duplicated from the use case rather than imported: the
// infrastructure layer must not depend on the application layer.
const reportsFolderName = "Reports"
// MoveToReportsFolder files a backfilled message out of the inbox, keeping the
// counters on both folders in step.
//
// The message keeps its identity and its blob; only the folder and the UID
// change, since a UID means nothing outside one folder.
func (s *DeliveredReportStore) MoveToReportsFolder(ctx context.Context, messageID uuid.UUID) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(ctx)
var mailboxID uuid.UUID
var sourceFolderID string
var wasUnread bool
err = tx.QueryRow(ctx, `
SELECT e.mailbox_id, e.folder_id::text,
NOT EXISTS (SELECT 1 FROM message_flags mf
WHERE mf.message_id = e.id
AND mf.received_at = e.received_at
AND mf.flag = '\Seen')
FROM email_messages e
WHERE e.id = $1 AND e.expunged_at IS NULL
`, messageID).Scan(&mailboxID, &sourceFolderID, &wasUnread)
if err != nil {
return fmt.Errorf("find message %s: %w", messageID, err)
}
var targetFolderID string
var targetUID int64
err = tx.QueryRow(ctx, `
SELECT id::text, uid_next FROM folders
WHERE mailbox_id = $1 AND LOWER(name) = LOWER($2)
LIMIT 1
FOR UPDATE
`, mailboxID, reportsFolderName).Scan(&targetFolderID, &targetUID)
if err != nil {
return fmt.Errorf("find the %s folder for mailbox %s: %w", reportsFolderName, mailboxID, err)
}
if targetFolderID == sourceFolderID {
return nil
}
var modseq int64
if err := tx.QueryRow(ctx, `
UPDATE folders SET uid_next = uid_next + 1, highest_modseq = highest_modseq + 1,
total_count = total_count + 1
WHERE id = $1
RETURNING highest_modseq
`, targetFolderID).Scan(&modseq); err != nil {
return err
}
if _, err := tx.Exec(ctx, `
UPDATE email_messages SET folder_id = $2, uid = $3, modseq = $4 WHERE id = $1
`, messageID, targetFolderID, targetUID, modseq); err != nil {
return err
}
// A backfilled report is machine-read mail: marking it seen is what stops
// it raising an unread badge in its new home.
if _, err := tx.Exec(ctx, `
INSERT INTO message_flags (message_id, received_at, flag)
SELECT e.id, e.received_at, '\Seen' FROM email_messages e WHERE e.id = $1
ON CONFLICT DO NOTHING
`, messageID); err != nil {
return err
}
unreadDelta := 0
if wasUnread {
unreadDelta = -1
}
if _, err := tx.Exec(ctx, `
UPDATE folders
SET total_count = GREATEST(0, total_count - 1),
unread_count = GREATEST(0, unread_count + $2),
highest_modseq = highest_modseq + 1
WHERE id = $1
`, sourceFolderID, unreadDelta); err != nil {
return err
}
return tx.Commit(ctx)
}
@@ -0,0 +1,355 @@
package postgres
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"lambdamail/protocols/internal/application/port"
"lambdamail/protocols/internal/application/usecase"
"lambdamail/protocols/internal/domain/entity"
"lambdamail/protocols/internal/infrastructure/diskstorage"
)
// End to end over a real database: a report arrives as mail, is parsed into
// the report tables, and is filed into Reports without raising an unread badge.
// The report Google actually sends, with the domain changed.
const liveTlsRptReport = `{"organization-name":"Google Inc.",` +
`"date-range":{"start-datetime":"2026-08-19T00:00:00Z","end-datetime":"2026-08-19T23:59:59Z"},` +
`"contact-info":"smtp-tls-reporting@google.com","report-id":"2026-08-19T00:00:00Z_example.test",` +
`"policies":[{"policy":{"policy-type":"no-policy-found","policy-domain":"example.test"},` +
`"summary":{"total-successful-session-count":2,"total-failure-session-count":0}}]}`
func reportMail(t *testing.T, filename string, content []byte) []byte {
t.Helper()
var gzBuf bytes.Buffer
gz := gzip.NewWriter(&gzBuf)
if _, err := gz.Write(content); err != nil {
t.Fatal(err)
}
if err := gz.Close(); err != nil {
t.Fatal(err)
}
encoded := base64.StdEncoding.EncodeToString(gzBuf.Bytes())
var b bytes.Buffer
b.WriteString("From: smtp-tls-reporting@google.com\r\n")
b.WriteString("Subject: Report Domain: example.test\r\n")
b.WriteString("MIME-Version: 1.0\r\n")
b.WriteString("Content-Type: multipart/mixed; boundary=\"sep\"\r\n\r\n")
b.WriteString("--sep\r\n")
b.WriteString("Content-Type: text/plain; charset=utf-8\r\n\r\nReport attached.\r\n")
b.WriteString("--sep\r\n")
b.WriteString("Content-Type: application/gzip; name=\"" + filename + "\"\r\n")
b.WriteString("Content-Disposition: attachment; filename=\"" + filename + "\"\r\n")
b.WriteString("Content-Transfer-Encoding: base64\r\n\r\n")
b.WriteString(encoded + "\r\n")
b.WriteString("--sep--\r\n")
return b.Bytes()
}
func TestDeliveredTlsRptReportIsIngestedAndFiled(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
mailboxID := seedMailbox(t, pool)
// The Reports folder migration 0008 adds for every mailbox.
if _, err := pool.Exec(ctx,
`INSERT INTO folders (mailbox_id, name, special_use) VALUES ($1, 'Reports', NULL)`,
uuid.MustParse(mailboxID)); err != nil {
t.Fatalf("seed Reports folder: %v", err)
}
reportRepo := NewReportRepository(pool)
inbound := usecase.NewProcessInboundEmailUseCase(
nil, &dbBlobStore{pool: pool, dir: t.TempDir()}, NewInboundMessageRepository(pool))
inbound.SetReportIngestor(
usecase.NewIngestDeliveredReportsUseCase(usecase.NewIngestReportsUseCase(reportRepo)))
reportID := "delivered-" + uuid.NewString()
payload := reportMail(t,
"google.com!example.test!1787097600!1787183999!001.json.gz",
[]byte(strings.Replace(liveTlsRptReport,
"2026-08-19T00:00:00Z_example.test", reportID, 1)))
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(),
`DELETE FROM tls_rpt_reports WHERE report_id = $1`, reportID)
})
if err := inbound.Handle(ctx, usecase.ProcessInboundEmailInput{
Sender: "smtp-tls-reporting@google.com",
Recipients: []port.MailboxRecord{{ID: uuid.MustParse(mailboxID)}},
RecipientAddresses: []string{"tlsrpt@example.test"},
Body: bytes.NewReader(payload),
}); err != nil {
t.Fatalf("Handle: %v", err)
}
// The report reached the table the admin console reads.
var stored int
if err := pool.QueryRow(ctx,
`SELECT COUNT(*) FROM tls_rpt_reports WHERE report_id = $1`, reportID).Scan(&stored); err != nil {
t.Fatalf("count stored reports: %v", err)
}
if stored != 1 {
t.Errorf("the report was not stored: %d rows", stored)
}
// And the message went to Reports, silently.
webmail := NewWebmailRepository(pool)
folders, err := webmail.ListFolders(ctx, mailboxID)
if err != nil {
t.Fatalf("list folders: %v", err)
}
for _, f := range folders {
switch f.Name {
case "Reports":
if f.TotalCount != 1 {
t.Errorf("Reports holds %d messages, want 1", f.TotalCount)
}
if f.UnreadCount != 0 {
t.Errorf("Reports shows %d unread; a machine-read report raises no badge", f.UnreadCount)
}
case "INBOX":
if f.TotalCount != 0 {
t.Errorf("the report landed in the inbox: %d messages", f.TotalCount)
}
}
}
}
// dbBlobStore writes a real message_blobs row so the delivery's foreign keys
// hold, without needing the disk storage driver in a database test.
type dbBlobStore struct {
pool *pgxpool.Pool
// dir is where the bytes actually land, so the real disk reader can read
// them back. Pointing storage_path at a file that does not exist would
// leave the read path untested.
dir string
}
func (s *dbBlobStore) Store(ctx context.Context, r io.Reader) (port.BlobRef, error) {
body, err := io.ReadAll(r)
if err != nil {
return port.BlobRef{}, err
}
sum := sha256.Sum256(body)
digest := hex.EncodeToString(sum[:])
// Deduplicated on the digest, as the real storage drivers are: identical
// bytes are one blob, and a second delivery of the same report reuses it.
id := uuid.New()
path := filepath.Join(s.dir, id.String()+".eml")
if err := os.WriteFile(path, body, 0o600); err != nil {
return port.BlobRef{}, err
}
if err := s.pool.QueryRow(ctx, `
INSERT INTO message_blobs (id, content_sha256, storage_driver, storage_path, size_bytes, ref_count)
VALUES ($1, $2, 'local', $3, $4, 0)
ON CONFLICT (content_sha256) DO UPDATE SET storage_path = EXCLUDED.storage_path
RETURNING id`,
id, digest, path, len(body)).Scan(&id); err != nil {
return port.BlobRef{}, err
}
return port.BlobRef{ID: id, SHA256: digest, SizeBytes: int64(len(body))}, nil
}
// A reporter that retries - or an SMTP retry after a slow commit - delivers
// the same report twice. Automatic ingestion makes that routine, where before
// reports were only ever written by hand through the API, so the same report
// arriving again must not double every number the console shows.
func TestIngestingTheSameReportTwiceStoresItOnce(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
repo := NewReportRepository(pool)
reportID := "retry-" + uuid.NewString()
report := func() *entity.TlsRptReport {
return &entity.TlsRptReport{
OrganizationName: "Google Inc.",
ReportID: reportID,
Domain: "example.test",
DateRangeBegin: time.Now().Add(-24 * time.Hour),
DateRangeEnd: time.Now(),
Policies: []entity.TlsRptPolicy{
{PolicyType: "no-policy-found", SuccessCount: 2, FailureCount: 0},
},
}
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(),
`DELETE FROM tls_rpt_reports WHERE report_id = $1`, reportID)
})
if err := repo.SaveTlsRptReport(ctx, report()); err != nil {
t.Fatalf("first save: %v", err)
}
if err := repo.SaveTlsRptReport(ctx, report()); err != nil {
t.Fatalf("a redelivered report must not error: %v", err)
}
var headers, policies int
if err := pool.QueryRow(ctx,
`SELECT COUNT(*) FROM tls_rpt_reports WHERE report_id = $1`, reportID).Scan(&headers); err != nil {
t.Fatal(err)
}
if headers != 1 {
t.Errorf("the same report is stored %d times", headers)
}
if err := pool.QueryRow(ctx, `
SELECT COUNT(*) FROM tls_rpt_report_policies p
JOIN tls_rpt_reports r ON r.id = p.report_id
WHERE r.report_id = $1`, reportID).Scan(&policies); err != nil {
t.Fatal(err)
}
if policies != 1 {
t.Errorf("the report's policies are stored %d times", policies)
}
}
// The same, for DMARC: its unique key includes domain_id, which this path
// never fills, and in Postgres a NULL never conflicts - so the key it had did
// not in fact prevent a duplicate.
func TestIngestingTheSameDmarcReportTwiceStoresItOnce(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
repo := NewReportRepository(pool)
reportID := "retry-" + uuid.NewString()
report := func() *entity.DmarcReport {
return &entity.DmarcReport{
OrgName: "google.com",
ReportID: reportID,
Domain: "example.test",
DateRangeBegin: time.Now().Add(-24 * time.Hour),
DateRangeEnd: time.Now(),
Records: []entity.DmarcRecord{
{SourceIP: "203.0.113.10", Count: 3, Disposition: "none", DKIMResult: "pass", SPFResult: "pass"},
},
}
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(),
`DELETE FROM dmarc_reports WHERE report_id = $1`, reportID)
})
if err := repo.SaveDmarcReport(ctx, report()); err != nil {
t.Fatalf("first save: %v", err)
}
if err := repo.SaveDmarcReport(ctx, report()); err != nil {
t.Fatalf("a redelivered report must not error: %v", err)
}
var headers int
if err := pool.QueryRow(ctx,
`SELECT COUNT(*) FROM dmarc_reports WHERE report_id = $1`, reportID).Scan(&headers); err != nil {
t.Fatal(err)
}
if headers != 1 {
t.Errorf("the same DMARC report is stored %d times", headers)
}
}
// The backfill, over a real database: a report that was delivered to the
// inbox before ingestion existed is parsed after the fact and filed away.
func TestBackfillParsesAReportAlreadySittingInTheInbox(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
mailboxID := seedMailbox(t, pool)
if _, err := pool.Exec(ctx,
`INSERT INTO folders (mailbox_id, name, special_use) VALUES ($1, 'Reports', NULL)`,
uuid.MustParse(mailboxID)); err != nil {
t.Fatalf("seed Reports folder: %v", err)
}
// Delivered the old way: straight into the inbox, unread, never parsed.
reportID := "backfill-" + uuid.NewString()
body := `{"organization-name":"Google Inc.",` +
`"date-range":{"start-datetime":"2026-08-19T00:00:00Z","end-datetime":"2026-08-19T23:59:59Z"},` +
`"report-id":"` + reportID + `",` +
`"policies":[{"policy":{"policy-type":"sts","policy-domain":"example.test"},` +
`"summary":{"total-successful-session-count":5,"total-failure-session-count":1}}]}`
payload := reportMail(t, "google.com!example.test!1!2!001.json.gz", []byte(body))
blobs := &dbBlobStore{pool: pool, dir: t.TempDir()}
blob, err := blobs.Store(ctx, bytes.NewReader(payload))
if err != nil {
t.Fatalf("store blob: %v", err)
}
uids, err := NewInboundMessageRepository(pool).PersistAll(ctx, []port.PersistInboundMessageInput{{
MailboxID: uuid.MustParse(mailboxID),
Blob: blob,
SenderAddress: "smtp-tls-reporting@google.com",
RecipientAddress: "tlsrpt@example.test",
TargetFolderName: "INBOX",
Subject: "Report Domain: example.test",
}})
if err != nil || len(uids) != 1 {
t.Fatalf("seed delivered report: %v", err)
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(),
`DELETE FROM tls_rpt_reports WHERE report_id = $1`, reportID)
})
store := NewDeliveredReportStore(pool)
uc := usecase.NewBackfillReportsUseCase(store, diskstorage.NewLocalDiskBlobReader(pool),
usecase.NewIngestDeliveredReportsUseCase(
usecase.NewIngestReportsUseCase(NewReportRepository(pool))))
summary, err := uc.Run(ctx, 0)
if err != nil {
t.Fatalf("backfill: %v", err)
}
if summary.Ingested < 1 {
t.Fatalf("the backlogged report was not ingested: %+v", summary)
}
var stored int
if err := pool.QueryRow(ctx,
`SELECT COUNT(*) FROM tls_rpt_reports WHERE report_id = $1`, reportID).Scan(&stored); err != nil {
t.Fatal(err)
}
if stored != 1 {
t.Errorf("the report reached the table %d times, want 1", stored)
}
// And it left the inbox.
webmail := NewWebmailRepository(pool)
folders, _ := webmail.ListFolders(ctx, mailboxID)
for _, f := range folders {
if f.Name == "INBOX" && f.TotalCount != 0 {
t.Errorf("the backfilled report is still in the inbox: %d messages", f.TotalCount)
}
if f.Name == "Reports" {
if f.TotalCount != 1 {
t.Errorf("Reports holds %d messages, want 1", f.TotalCount)
}
if f.UnreadCount != 0 {
t.Errorf("the backfilled report raised an unread badge: %d", f.UnreadCount)
}
}
}
// Running it again finds nothing left to do.
second, err := uc.Run(ctx, 0)
if err != nil {
t.Fatalf("second run: %v", err)
}
if second.Ingested != 0 {
t.Errorf("a second pass re-ingested %d reports; it should find none", second.Ingested)
}
}
@@ -25,16 +25,35 @@ func (r *ReportRepository) SaveDmarcReport(ctx context.Context, report *entity.D
}
defer tx.Rollback(ctx)
// Idempotent on (org_name, report_id): the reporter's own identifier for
// the report. Reports are now parsed out of inbound mail automatically, so
// a redelivery - an SMTP retry, or a reporter simply sending again - is
// routine, and storing the same report twice would double every number the
// admin console shows.
//
// DO UPDATE rather than DO NOTHING so the existing row's id comes back and
// the records below can be reconciled against it; DO NOTHING returns no
// row at all when it conflicts.
var reportID uuid.UUID
err = tx.QueryRow(ctx, `
INSERT INTO dmarc_reports (org_name, report_id, domain, date_range_begin, date_range_end)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (org_name, report_id)
DO UPDATE SET domain = EXCLUDED.domain
RETURNING id
`, report.OrgName, report.ReportID, report.Domain, report.DateRangeBegin, report.DateRangeEnd).Scan(&reportID)
if err != nil {
return fmt.Errorf("insert dmarc report header: %w", err)
}
// The records are replaced, not appended: on a redelivery the report's
// previous rows are still here, and adding another set would double every
// count derived from them.
if _, err := tx.Exec(ctx,
`DELETE FROM dmarc_report_records WHERE report_id = $1`, reportID); err != nil {
return fmt.Errorf("clear dmarc records: %w", err)
}
for _, rec := range report.Records {
_, err := tx.Exec(ctx, `
INSERT INTO dmarc_report_records (report_id, source_ip, count, disposition, dkim_result, spf_result, header_from)
@@ -55,16 +74,24 @@ func (r *ReportRepository) SaveTlsRptReport(ctx context.Context, report *entity.
}
defer tx.Rollback(ctx)
// Idempotent for the same reason as the DMARC writer above.
var reportID uuid.UUID
err = tx.QueryRow(ctx, `
INSERT INTO tls_rpt_reports (organization_name, report_id, domain, date_range_begin, date_range_end)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (organization_name, report_id)
DO UPDATE SET domain = EXCLUDED.domain
RETURNING id
`, report.OrganizationName, report.ReportID, report.Domain, report.DateRangeBegin, report.DateRangeEnd).Scan(&reportID)
if err != nil {
return fmt.Errorf("insert tls-rpt report header: %w", err)
}
if _, err := tx.Exec(ctx,
`DELETE FROM tls_rpt_report_policies WHERE report_id = $1`, reportID); err != nil {
return fmt.Errorf("clear tls-rpt policies: %w", err)
}
for _, pol := range report.Policies {
_, err := tx.Exec(ctx, `
INSERT INTO tls_rpt_report_policies (report_id, policy_type, success_count, failure_count)
@@ -0,0 +1,191 @@
package httppresentation
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/google/uuid"
"lambdamail/protocols/internal/application/port"
"lambdamail/protocols/internal/application/usecase"
)
// The event stream tells a browser its mailbox changed. It is the one place
// where one account's activity is pushed out over another's connection if the
// filtering is wrong, and none of it was covered.
func newTestHub(t *testing.T, store *fakeMailStore) *EventHub {
t.Helper()
uc := usecase.NewWebmailUseCase(store, &fakeBlobReader{}, nil, nil, "mail.example.test")
return NewEventHub(testVerifier(interopSecret), uc)
}
// The hub matches on the aggregate's UUID string, so subscribers are keyed by
// real UUIDs rather than the readable names the mail API fakes use.
var (
mailboxOne = uuid.New()
mailboxTwo = uuid.New()
)
// subscribeTo registers a subscriber directly, which is what the WebSocket
// handler does after authenticating. Driving a real upgrade would test the
// library; this tests the routing decision, which is ours.
func subscribeTo(hub *EventHub, mailboxID string, buffer int) *subscriber {
sub := &subscriber{mailboxID: mailboxID, events: make(chan EventMessage, buffer)}
hub.add(sub)
return sub
}
// The check that keeps one mailbox's activity off another's connection.
func TestEventsReachOnlyTheirOwnMailbox(t *testing.T) {
hub := newTestHub(t, storeWithMail())
mine := subscribeTo(hub, mailboxOne.String(), 4)
theirs := subscribeTo(hub, mailboxTwo.String(), 4)
hub.Publish(port.OutboxEvent{
EventType: "EmailReceived",
AggregateID: mailboxOne,
Payload: []byte(`{"uid":7}`),
})
select {
case msg := <-mine.events:
if msg.Type != "EmailReceived" {
t.Errorf("got event type %q", msg.Type)
}
default:
t.Error("the mailbox's own event never arrived")
}
select {
case msg := <-theirs.events:
t.Errorf("another mailbox's event was delivered: %+v", msg)
default:
}
}
// A subscriber that has stopped reading must be dropped, never allowed to
// stall the relay that every other connection shares.
func TestASlowSubscriberIsDroppedRatherThanBlocking(t *testing.T) {
hub := newTestHub(t, storeWithMail())
// Buffer of one, then filled, so the next publish has nowhere to go.
stalled := subscribeTo(hub, mailboxOne.String(), 1)
stalled.events <- EventMessage{Type: "filler"}
done := make(chan struct{})
go func() {
hub.Publish(port.OutboxEvent{
EventType: "EmailReceived",
AggregateID: mailboxOne,
Payload: []byte(`{}`),
})
close(done)
}()
select {
case <-done:
case <-context.Background().Done():
t.Fatal("Publish blocked on a subscriber that was not reading")
}
}
func TestSubscriberCountFollowsAddAndRemove(t *testing.T) {
hub := newTestHub(t, storeWithMail())
if hub.SubscriberCount() != 0 {
t.Fatalf("a new hub has %d subscribers", hub.SubscriberCount())
}
sub := subscribeTo(hub, mailboxOne.String(), 1)
if hub.SubscriberCount() != 1 {
t.Errorf("after one subscriber: %d", hub.SubscriberCount())
}
hub.remove(sub)
if hub.SubscriberCount() != 0 {
t.Errorf("after removing it: %d", hub.SubscriberCount())
}
}
// Publishing with nobody listening must be a no-op, not a panic: the relay
// runs whether or not a browser happens to be open.
func TestPublishWithNoSubscribersIsHarmless(t *testing.T) {
hub := newTestHub(t, storeWithMail())
hub.Publish(port.OutboxEvent{
EventType: "EmailReceived",
AggregateID: uuid.New(),
Payload: []byte(`{}`),
})
}
// --- the upgrade's own authentication ------------------------------------
// The session is verified before the upgrade, so an unauthenticated caller
// gets a plain 401 and never reaches the WebSocket machinery.
func TestEventStreamRefusesAnUnauthenticatedUpgrade(t *testing.T) {
hub := newTestHub(t, storeWithMail())
rec := httptest.NewRecorder()
hub.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/events", nil))
if rec.Code != http.StatusUnauthorized {
t.Errorf("an unauthenticated upgrade answered %d, want 401", rec.Code)
}
}
func TestEventStreamRefusesAnAdminSession(t *testing.T) {
hub := newTestHub(t, storeWithMail())
req := httptest.NewRequest(http.MethodGet, "/api/v1/events", nil)
req.AddCookie(&http.Cookie{Name: "lm_user_session", Value: adminSurfaceToken(t)})
rec := httptest.NewRecorder()
hub.ServeHTTP(rec, req)
if rec.Code != http.StatusUnauthorized {
t.Errorf("an admin session reached the event stream: %d", rec.Code)
}
}
// A valid session for an address this server no longer serves must not open a
// stream either - that is a deleted account whose token has not expired yet.
func TestEventStreamRefusesASessionWithNoMailbox(t *testing.T) {
store := storeWithMail()
delete(store.addresses, sessionOwner)
hub := newTestHub(t, store)
req := httptest.NewRequest(http.MethodGet, "/api/v1/events", nil)
req.AddCookie(&http.Cookie{Name: "lm_user_session", Value: interopSession})
rec := httptest.NewRecorder()
hub.ServeHTTP(rec, req)
if rec.Code != http.StatusUnauthorized {
t.Errorf("a session with no mailbox answered %d, want 401", rec.Code)
}
}
// The payload has to reach the browser unchanged: the page reads the UID out
// of it to decide what to refresh.
func TestEventCarriesItsPayloadThrough(t *testing.T) {
hub := newTestHub(t, storeWithMail())
sub := subscribeTo(hub, mailboxOne.String(), 2)
hub.Publish(port.OutboxEvent{
EventType: "EmailReceived",
AggregateID: mailboxOne,
Payload: []byte(`{"recipient":"user@example.test","uid":42}`),
})
msg := <-sub.events
var payload struct {
Recipient string `json:"recipient"`
UID int `json:"uid"`
}
if err := json.Unmarshal(msg.Payload, &payload); err != nil {
t.Fatalf("payload did not survive: %v", err)
}
if payload.UID != 42 || payload.Recipient != "user@example.test" {
t.Errorf("payload arrived as %+v", payload)
}
}
@@ -0,0 +1,539 @@
package httppresentation
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/google/uuid"
"lambdamail/protocols/internal/application/port"
"lambdamail/protocols/internal/application/usecase"
"lambdamail/protocols/internal/infrastructure/postgres"
)
// The webmail's HTTP surface: the only part of this server a browser talks to
// directly, and the one that decides whose mail a session may touch.
//
// None of it was covered. Every one of these asserts something a mistake here
// would cost: another account's mail, an attachment rendered as a page on this
// origin, or a sender address chosen by the request rather than the session.
// --- test doubles --------------------------------------------------------
// fakeMailStore records what the API asked for, so a handler can be checked
// on the arguments it passed rather than only on its status code.
type fakeMailStore struct {
// byMailbox is the mail each mailbox owns, keyed by mailbox ID.
byMailbox map[string][]port.WebmailMessage
addresses map[string]string // address -> mailbox ID
seenCalls []string
deletedCalls []string
movedCalls []string
expungeErr error
moveErr error
blob uuid.UUID
}
func (f *fakeMailStore) FindMailboxIDByAddress(_ context.Context, address string) (string, error) {
id, ok := f.addresses[address]
if !ok {
return "", postgres.ErrMessageNotFound
}
return id, nil
}
func (f *fakeMailStore) ListFolders(_ context.Context, mailboxID string) ([]port.WebmailFolder, error) {
return []port.WebmailFolder{{
ID: "f1", Name: "INBOX", SpecialUse: "inbox",
TotalCount: len(f.byMailbox[mailboxID]),
}}, nil
}
func (f *fakeMailStore) ListMessages(
_ context.Context, mailboxID, _, _ string, _, _ int,
) ([]port.WebmailMessage, error) {
return f.byMailbox[mailboxID], nil
}
func (f *fakeMailStore) GetMessageBlob(
_ context.Context, mailboxID, _ string, uid uint32,
) (uuid.UUID, error) {
for _, m := range f.byMailbox[mailboxID] {
if m.UID == uid {
return f.blob, nil
}
}
// Exactly what the real repository does for a UID this mailbox does not
// own: not found, indistinguishable from one that does not exist.
return uuid.Nil, postgres.ErrMessageNotFound
}
func (f *fakeMailStore) MarkSeen(_ context.Context, mailboxID, folder string, uid uint32, seen bool) error {
f.seenCalls = append(f.seenCalls, mailboxID+"/"+folder)
return nil
}
func (f *fakeMailStore) Expunge(_ context.Context, mailboxID, folder string, uid uint32) error {
if f.expungeErr != nil {
return f.expungeErr
}
f.expungeErr = nil
f.deletedCalls = append(f.deletedCalls, mailboxID+"/"+folder)
return nil
}
func (f *fakeMailStore) MoveToTrash(
_ context.Context, mailboxID, folder string, uid uint32,
) (uint32, error) {
if f.moveErr != nil {
return 0, f.moveErr
}
f.movedCalls = append(f.movedCalls, mailboxID+"/"+folder)
return uid, nil
}
type fakeBlobReader struct{ raw []byte }
func (f *fakeBlobReader) ReadByID(context.Context, uuid.UUID) ([]byte, error) {
return f.raw, nil
}
// --- harness -------------------------------------------------------------
// newMailAPI builds the API over fakes, pinned to an instant inside the
// captured tokens' validity window so these tests never age out.
func newMailAPI(t *testing.T, store *fakeMailStore, raw []byte) *mailAPI {
t.Helper()
uc := usecase.NewWebmailUseCase(store, &fakeBlobReader{raw: raw}, nil, nil, "mail.example.test")
return &mailAPI{useCase: uc, sessions: testVerifier(interopSecret)}
}
// The session in interopSession belongs to this address.
const sessionOwner = "user@example.test"
func request(method, target string, body string) *http.Request {
req := httptest.NewRequest(method, target, strings.NewReader(body))
req.Header.Set("Authorization", "Bearer "+interopSession)
req.Header.Set("Content-Type", "application/json")
return req
}
func storeWithMail() *fakeMailStore {
return &fakeMailStore{
addresses: map[string]string{sessionOwner: "mailbox-1", "other@example.test": "mailbox-2"},
byMailbox: map[string][]port.WebmailMessage{
"mailbox-1": {{UID: 1, Subject: "mine"}},
"mailbox-2": {{UID: 99, Subject: "somebody else's"}},
},
blob: uuid.New(),
}
}
// --- authentication ------------------------------------------------------
func TestMailApiRefusesARequestWithNoSession(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
for name, req := range map[string]*http.Request{
"folders": httptest.NewRequest(http.MethodGet, "/api/v1/mail/folders", nil),
"messages": httptest.NewRequest(http.MethodGet, "/api/v1/mail/messages", nil),
"seen": httptest.NewRequest(http.MethodPost, "/api/v1/mail/seen", strings.NewReader(`{"uid":1}`)),
"delete": httptest.NewRequest(http.MethodPost, "/api/v1/mail/delete", strings.NewReader(`{"uid":1}`)),
} {
rec := httptest.NewRecorder()
switch name {
case "folders":
api.handleFolders(rec, req)
case "messages":
api.handleMessages(rec, req)
case "seen":
api.handleSeen(rec, req)
case "delete":
api.handleDelete(rec, req)
}
if rec.Code != http.StatusUnauthorized {
t.Errorf("%s answered %d without a session, want 401", name, rec.Code)
}
}
}
// An admin session must not reach the mail API. The two surfaces are separated
// by audience precisely so a console token cannot read a mailbox.
func TestMailApiRefusesAnAdminSession(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
req := httptest.NewRequest(http.MethodGet, "/api/v1/mail/folders", nil)
req.Header.Set("Authorization", "Bearer "+adminSurfaceToken(t))
rec := httptest.NewRecorder()
api.handleFolders(rec, req)
if rec.Code != http.StatusUnauthorized {
t.Errorf("an admin token reached the mail API: %d", rec.Code)
}
}
// The half-issued token from between the password and the second factor is not
// a session and must not be accepted as one.
func TestMailApiRefusesAnMfaChallengeToken(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
req := httptest.NewRequest(http.MethodGet, "/api/v1/mail/folders", nil)
req.Header.Set("Authorization", "Bearer "+interopChallenge)
rec := httptest.NewRecorder()
api.handleFolders(rec, req)
if rec.Code != http.StatusUnauthorized {
t.Errorf("an MFA challenge token was accepted as a session: %d", rec.Code)
}
}
// A tampered signature must fail, or the secret is doing nothing.
func TestMailApiRefusesATamperedToken(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
req := httptest.NewRequest(http.MethodGet, "/api/v1/mail/folders", nil)
req.Header.Set("Authorization", "Bearer "+interopSession[:len(interopSession)-4]+"AAAA")
rec := httptest.NewRecorder()
api.handleFolders(rec, req)
if rec.Code != http.StatusUnauthorized {
t.Errorf("a token with a broken signature was accepted: %d", rec.Code)
}
}
// The cookie is the browser's normal path and must work like the header.
func TestMailApiAcceptsTheSessionCookie(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
req := httptest.NewRequest(http.MethodGet, "/api/v1/mail/folders", nil)
req.AddCookie(&http.Cookie{Name: "lm_user_session", Value: interopSession})
rec := httptest.NewRecorder()
api.handleFolders(rec, req)
if rec.Code != http.StatusOK {
t.Errorf("the session cookie was not accepted: %d", rec.Code)
}
}
// --- isolation between accounts -----------------------------------------
// The mailbox is resolved from the session's address, never from the request.
// This is the check that keeps one account out of another's mail.
func TestMailApiServesOnlyTheSessionsOwnMailbox(t *testing.T) {
store := storeWithMail()
api := newMailAPI(t, store, nil)
rec := httptest.NewRecorder()
api.handleMessages(rec, request(http.MethodGet, "/api/v1/mail/messages", ""))
var got []port.WebmailMessage
if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
for _, m := range got {
if m.Subject == "somebody else's" {
t.Fatal("the API served another mailbox's mail")
}
}
if len(got) != 1 || got[0].Subject != "mine" {
t.Errorf("got %+v, want only this session's own message", got)
}
}
// Naming another mailbox's UID must read as not-found, not as that message.
func TestMailApiCannotReadAnotherMailboxesMessageByUid(t *testing.T) {
store := storeWithMail()
api := newMailAPI(t, store, []byte("Subject: secret\r\n\r\nbody"))
rec := httptest.NewRecorder()
// UID 99 exists, but in mailbox-2.
api.handleMessage(rec, request(http.MethodGet, "/api/v1/mail/message/99", ""))
if rec.Code != http.StatusNotFound {
t.Errorf("reading another mailbox's UID answered %d, want 404", rec.Code)
}
if strings.Contains(rec.Body.String(), "secret") {
t.Error("another mailbox's message body was served")
}
}
// --- reading -------------------------------------------------------------
// Opening a message marks it read; that is what moves the unread counter.
func TestOpeningAMessageMarksItRead(t *testing.T) {
store := storeWithMail()
api := newMailAPI(t, store, []byte("Subject: hi\r\n\r\nbody"))
rec := httptest.NewRecorder()
api.handleMessage(rec, request(http.MethodGet, "/api/v1/mail/message/1?folder=inbox", ""))
if rec.Code != http.StatusOK {
t.Fatalf("reading own message answered %d", rec.Code)
}
if len(store.seenCalls) != 1 || store.seenCalls[0] != "mailbox-1/inbox" {
t.Errorf("marked seen on %v, want exactly [mailbox-1/inbox]", store.seenCalls)
}
}
// The folder defaults rather than 400-ing, because the list view omits it.
func TestMessageListDefaultsToTheInbox(t *testing.T) {
store := storeWithMail()
api := newMailAPI(t, store, nil)
rec := httptest.NewRecorder()
api.handleSeen(rec, request(http.MethodPost, "/api/v1/mail/seen", `{"uid":1,"seen":true}`))
if rec.Code != http.StatusOK {
t.Fatalf("answered %d", rec.Code)
}
if len(store.seenCalls) != 1 || store.seenCalls[0] != "mailbox-1/inbox" {
t.Errorf("the folder did not default to inbox: %v", store.seenCalls)
}
}
// --- deleting ------------------------------------------------------------
func TestDeleteMovesToTrashFromAnOrdinaryFolder(t *testing.T) {
store := storeWithMail()
api := newMailAPI(t, store, nil)
rec := httptest.NewRecorder()
api.handleDelete(rec, request(http.MethodPost, "/api/v1/mail/delete", `{"folder":"inbox","uid":1}`))
if rec.Code != http.StatusOK {
t.Fatalf("delete answered %d: %s", rec.Code, rec.Body)
}
// The mailbox is asserted, not just the call: counting calls alone would
// pass just as happily if the delete acted on somebody else's mailbox.
if len(store.movedCalls) != 1 || store.movedCalls[0] != "mailbox-1/inbox" {
t.Errorf("moved %v, want exactly [mailbox-1/inbox]", store.movedCalls)
}
if len(store.deletedCalls) != 0 {
t.Errorf("deleting from the inbox destroyed the message: %v", store.deletedCalls)
}
}
func TestDeleteFromTrashExpungesForGood(t *testing.T) {
store := storeWithMail()
store.moveErr = port.ErrAlreadyInTrash
api := newMailAPI(t, store, nil)
rec := httptest.NewRecorder()
api.handleDelete(rec, request(http.MethodPost, "/api/v1/mail/delete", `{"folder":"trash","uid":1}`))
if rec.Code != http.StatusOK {
t.Fatalf("delete from trash answered %d: %s", rec.Code, rec.Body)
}
if len(store.deletedCalls) != 1 || store.deletedCalls[0] != "mailbox-1/trash" {
t.Errorf("expunged %v, want exactly [mailbox-1/trash]", store.deletedCalls)
}
}
// A delete with no UID is a bug in the caller, not a request to delete UID 0.
func TestDeleteRejectsAMissingUid(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
rec := httptest.NewRecorder()
api.handleDelete(rec, request(http.MethodPost, "/api/v1/mail/delete", `{"folder":"inbox"}`))
if rec.Code != http.StatusBadRequest {
t.Errorf("a delete with no UID answered %d, want 400", rec.Code)
}
}
func TestDeleteRejectsMalformedJson(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
rec := httptest.NewRecorder()
api.handleDelete(rec, request(http.MethodPost, "/api/v1/mail/delete", `{not json`))
if rec.Code != http.StatusBadRequest {
t.Errorf("malformed JSON answered %d, want 400", rec.Code)
}
}
// --- error mapping -------------------------------------------------------
// Each domain error has to reach the browser as something the composer can act
// on. A send limit that arrives as a 500 tells the user to try again, which is
// the one thing that will not work.
func TestDomainErrorsMapToUsefulStatusCodes(t *testing.T) {
cases := []struct {
err error
want int
}{
{postgres.ErrMessageNotFound, http.StatusNotFound},
{usecase.ErrNoSuchMailbox, http.StatusNotFound},
{usecase.ErrNoRecipients, http.StatusBadRequest},
{usecase.ErrSendLimitExceeded, http.StatusTooManyRequests},
{usecase.ErrSenderNotOwned, http.StatusForbidden},
{usecase.ErrDraftsUnavailable, http.StatusServiceUnavailable},
{errors.New("something unexpected"), http.StatusInternalServerError},
}
api := newMailAPI(t, storeWithMail(), nil)
for _, tc := range cases {
rec := httptest.NewRecorder()
api.fail(rec, tc.err)
if rec.Code != tc.want {
t.Errorf("%v mapped to %d, want %d", tc.err, rec.Code, tc.want)
}
// The internals must not travel to the browser with it.
if strings.Contains(rec.Body.String(), "something unexpected") {
t.Errorf("an internal error message leaked to the client: %s", rec.Body)
}
}
}
// adminSurfaceToken mints a token for the admin surface under the same secret,
// to prove the audience check is what refuses it rather than the signature.
func adminSurfaceToken(t *testing.T) string {
t.Helper()
return mintSession(t, WebSession{
Subject: "mb-1", Email: sessionOwner, Role: "SUPER_ADMIN",
Surface: "admin", Audience: "lambdamail:admin",
Purpose: "session", MfaSatisfied: true,
IssuedAt: interopIssuedAt,
ExpiresAt: interopIssuedAt + 3600,
})
}
var _ = time.Now
// --- composing -----------------------------------------------------------
// The sender is taken from the session and never from the request body.
// Letting the body choose From is how a webmail becomes an open relay for
// spoofed mail, signed with this domain's own DKIM key.
func TestComposeAlwaysSendsAsTheSessionsOwnAddress(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
rec := httptest.NewRecorder()
req := request(http.MethodPost, "/api/v1/mail/send", `{
"from":"ceo@bank.example",
"to":["victim@example.test"],
"subject":"Wire transfer",
"html":"<p>urgent</p>"
}`)
input, ok := api.decodeCompose(rec, req, sessionOwner)
if !ok {
t.Fatalf("decode refused a valid body: %s", rec.Body)
}
if input.From != sessionOwner {
t.Errorf("From is %q; the request body chose the sender", input.From)
}
}
// Bcc has to survive decoding, or a blind copy is silently dropped and nobody
// is told the message did not reach them.
func TestComposeKeepsBlindCopies(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
rec := httptest.NewRecorder()
input, ok := api.decodeCompose(rec, request(http.MethodPost, "/api/v1/mail/send", `{
"to":["a@example.test"],"cc":["b@example.test"],"bcc":["c@example.test"],"text":"hi"
}`), sessionOwner)
if !ok {
t.Fatalf("decode refused: %s", rec.Body)
}
if len(input.Bcc) != 1 || input.Bcc[0] != "c@example.test" {
t.Errorf("Bcc arrived as %v", input.Bcc)
}
if len(input.Cc) != 1 {
t.Errorf("Cc arrived as %v", input.Cc)
}
}
// The composer sends "html"; the field was once read as "body", and every
// message went out blank. The legacy name still has to work for anything
// written against the old shape.
func TestComposeAcceptsBothTheCurrentAndLegacyBodyFields(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
rec := httptest.NewRecorder()
current, ok := api.decodeCompose(rec, request(http.MethodPost, "/x",
`{"to":["a@example.test"],"html":"<p>real</p>"}`), sessionOwner)
if !ok || current.HTML != "<p>real</p>" {
t.Errorf("the html field did not survive: %+v", current)
}
rec = httptest.NewRecorder()
legacy, ok := api.decodeCompose(rec, request(http.MethodPost, "/x",
`{"to":["a@example.test"],"body":"plain words"}`), sessionOwner)
if !ok || legacy.Text != "plain words" {
t.Errorf("the legacy body field was dropped: %+v", legacy)
}
}
// The draft a sent message came from has to reach the use case, or the draft
// outlives the message it became.
func TestComposeCarriesTheDraftItCameFrom(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
rec := httptest.NewRecorder()
input, ok := api.decodeCompose(rec, request(http.MethodPost, "/api/v1/mail/send",
`{"to":["a@example.test"],"text":"hi","draft_uid":77}`), sessionOwner)
if !ok {
t.Fatalf("decode refused: %s", rec.Body)
}
if input.DraftUID != 77 {
t.Errorf("DraftUID is %d, want 77", input.DraftUID)
}
}
// The draft decode reads the body once and has to get both halves out of it.
func TestDraftDecodeReadsBothTheMessageAndTheUidItReplaces(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
rec := httptest.NewRecorder()
input, replaceUID, ok := api.decodeDraft(rec, request(http.MethodPost, "/api/v1/mail/draft",
`{"to":["a@example.test"],"subject":"half","html":"<p>writ</p>","replace_uid":12}`), sessionOwner)
if !ok {
t.Fatalf("decode refused: %s", rec.Body)
}
if replaceUID != 12 {
t.Errorf("replace_uid is %d, want 12", replaceUID)
}
if input.Subject != "half" || input.HTML != "<p>writ</p>" {
t.Errorf("the draft body was lost: %+v", input)
}
if input.From != sessionOwner {
t.Errorf("a draft's From is %q, want the session's address", input.From)
}
}
// A draft with no replace_uid is the first save of a message and must still
// decode - it simply supersedes nothing.
func TestDraftDecodeToleratesAMissingReplaceUid(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
rec := httptest.NewRecorder()
_, replaceUID, ok := api.decodeDraft(rec, request(http.MethodPost, "/api/v1/mail/draft",
`{"subject":"first save"}`), sessionOwner)
if !ok {
t.Fatalf("the first save of a draft was refused: %s", rec.Body)
}
if replaceUID != 0 {
t.Errorf("replace_uid is %d, want 0", replaceUID)
}
}
func TestComposeRejectsMalformedJson(t *testing.T) {
api := newMailAPI(t, storeWithMail(), nil)
rec := httptest.NewRecorder()
if _, ok := api.decodeCompose(rec, request(http.MethodPost, "/x", `{"to":`), sessionOwner); ok {
t.Error("malformed JSON was accepted")
}
if rec.Code != http.StatusBadRequest {
t.Errorf("answered %d, want 400", rec.Code)
}
}
@@ -1,6 +1,10 @@
package httppresentation
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
"testing"
"time"
@@ -91,3 +95,44 @@ func TestWebSessionVerifier_RejectsExpiredToken(t *testing.T) {
t.Fatalf("error = %v, want ErrTokenExpired", err)
}
}
// mintSession signs a session token the way the auth service does, so a test
// can build a claim set the captured tokens do not cover - another surface, an
// expired window, a different address.
//
// The captured tokens above stay the interop check: this only produces the
// variations, and produces them through the same algorithm.
func mintSession(t *testing.T, claims WebSession) string {
t.Helper()
header := base64.RawURLEncoding.EncodeToString(
[]byte(`{"alg":"HS256","typ":"JWT"}`))
body, err := json.Marshal(claims)
if err != nil {
t.Fatalf("marshal claims: %v", err)
}
payload := base64.RawURLEncoding.EncodeToString(body)
mac := hmac.New(sha256.New, []byte(interopSecret))
mac.Write([]byte(header + "." + payload))
signature := base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
return header + "." + payload + "." + signature
}
// The minted tokens have to verify, or every test built on them is asserting
// against a signature this server would reject for the wrong reason.
func TestMintedSessionVerifiesLikeARealOne(t *testing.T) {
token := mintSession(t, WebSession{
Subject: "mb-1", Email: "user@example.test", Surface: "user",
Audience: "lambdamail:user", Purpose: "session", MfaSatisfied: true,
IssuedAt: interopIssuedAt, ExpiresAt: interopIssuedAt + 3600,
})
session, err := testVerifier(interopSecret).RequireSurface(token, "user")
if err != nil {
t.Fatalf("a locally minted session was rejected: %v", err)
}
if session.Email != "user@example.test" {
t.Errorf("Email = %q", session.Email)
}
}