feat: add test database setup and teardown for end-to-end tests, ensuring unique and non-ephemeral ports

Makefile

@@ -1,4 +1,4 @@
-.PHONY: up down logs test lint migrate-up migrate-down gen-dev-cert seed preflight create-admin reset-password reset-mfa
+.PHONY: up down logs test test-db-up test-db-down lint migrate-up migrate-down gen-dev-cert seed preflight create-admin reset-password reset-mfa
 
 # Local development builds from source; production pulls what CI published.
 up:
@@ -10,12 +10,45 @@ down:
 logs:
 	docker compose logs -f
 
+# TEST_DATABASE_URL is what decides whether the suites that assert against
+# real rows actually run. Without it the Go postgres tests t.Skip and the auth
+# suite skips 35 of its 72 - and both report success, so "make test" can print
+# a clean run while the tests that would have caught a schema change never
+# executed. That is not hypothetical: a folder added to createMailbox passed
+# here and failed in CI, which does set the variable.
 test:
+	@if [ -z "$$TEST_DATABASE_URL" ]; then 		echo "=============================================================="; 		echo "TEST_DATABASE_URL is not set."; 		echo "The database-backed tests will SKIP, and the suites will still"; 		echo "report success. CI sets it, so a green run here does not mean"; 		echo "a green run there. Start one with:"; 		echo "  make test-db-up"; 		echo "=============================================================="; 	fi
 	cd services/protocols && go test ./...
 	cd services/auth && npm test
 	cd services/storage && npm test
 	cd services/webmail && npm test
 
+# Brings up a throwaway Postgres with every migration applied, and prints the
+# URL to export. Matches what CI provisions, so "make test" with this exported
+# runs the same set of tests CI does.
+test-db-up:
+	docker rm -f lambdamail-test-db 2>/dev/null || true
+	docker run -d --name lambdamail-test-db \
+		-e POSTGRES_USER=lambdamail_test \
+		-e POSTGRES_PASSWORD=lambdamail_test \
+		-e POSTGRES_DB=lambdamail_test \
+		-p 55432:5432 postgres:16-alpine
+	@# pg_isready alone is not enough: the postgres entrypoint brings the
+	@# server up once to run its init scripts and then restarts it, so there is
+	@# a window where it answers ready and then drops the connection. A real
+	@# query is what proves it is actually serving.
+	@until docker exec lambdamail-test-db psql -U lambdamail_test -d lambdamail_test -c "SELECT 1" >/dev/null 2>&1; do sleep 1; done
+	docker run --rm --network host -v "$(PWD)/migrations:/migrations" \
+		migrate/migrate:v4.17.1 -path=/migrations \
+		-database "postgres://lambdamail_test:lambdamail_test@localhost:55432/lambdamail_test?sslmode=disable" up
+	@echo ""
+	@echo "Ready. Export this, then run make test:"
+	@echo "  export TEST_DATABASE_URL='postgres://lambdamail_test:lambdamail_test@localhost:55432/lambdamail_test?sslmode=disable'"
+	@echo "  export LAMBDAMAIL_MASTER_KEY='local-master-key-at-least-16-chars'"
+
+test-db-down:
+	docker rm -f lambdamail-test-db 2>/dev/null || true
+
 lint:
 	cd services/protocols && go vet ./...
 	node scripts/lang-lint.mjs

services/auth/src/router.test.ts

@@ -412,10 +412,17 @@ describeDb("auth API against a real database", () => {
     expect(created.status).toBe(201);
 
     // Delivery depends on these existing: a mailbox with no Junk folder used
-    // to fail the spam path outright.
-    const folders = await query<{ special_use: string }>(
-      `SELECT special_use FROM folders WHERE mailbox_id = $1`, [created.body.id]);
-    expect(folders.map((f) => f.special_use).sort()).toEqual(
+    // to fail the spam path outright, and one with no Reports folder files
+    // every delivered DMARC and TLS-RPT report into the inbox instead.
+    //
+    // Asserted on the name rather than the special-use role: IMAP defines no
+    // role for Reports, so its role is NULL and a list of roles cannot say
+    // whether the folder is there.
+    const folders = await query<{ name: string; special_use: string | null }>(
+      `SELECT name, special_use FROM folders WHERE mailbox_id = $1`, [created.body.id]);
+    expect(folders.map((f) => f.name).sort()).toEqual(
+      ["Archive", "Drafts", "INBOX", "Junk", "Reports", "Sent", "Trash"]);
+    expect(folders.map((f) => f.special_use).filter(Boolean).sort()).toEqual(
       ["archive", "drafts", "inbox", "junk", "sent", "trash"]);
 
     const audit = await get("/api/v1/admin/audit", admin);

services/protocols/internal/e2e/antispam_antivirus_e2e_test.go

@@ -30,7 +30,7 @@ func TestAntispamAntivirusEndToEnd(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54338).
+		Port(15438).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -38,7 +38,7 @@ func TestAntispamAntivirusEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54338/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15438/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect pool: %v", err)

services/protocols/internal/e2e/dkim_rotation_e2e_test.go

@@ -30,7 +30,7 @@ func TestDkimRotationLeavesAUsableKey(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54349).
+		Port(15449).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -38,7 +38,7 @@ func TestDkimRotationLeavesAUsableKey(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	pool, err := postgres.NewPool(ctx, "postgres://postgres:postgres@localhost:54349/postgres?sslmode=disable")
+	pool, err := postgres.NewPool(ctx, "postgres://postgres:postgres@localhost:15449/postgres?sslmode=disable")
 	if err != nil {
 		t.Fatal(err)
 	}

services/protocols/internal/e2e/dns_sync_e2e_test.go

@@ -21,7 +21,7 @@ func TestDnsSyncAndSystemAliasesEndToEnd(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54336).
+		Port(15436).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -29,7 +29,7 @@ func TestDnsSyncAndSystemAliasesEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54336/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15436/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect pool: %v", err)

services/protocols/internal/e2e/events_ws_e2e_test.go

@@ -31,7 +31,7 @@ func TestEventStreamPushesDeliveredMail(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54348).
+		Port(15448).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -39,7 +39,7 @@ func TestEventStreamPushesDeliveredMail(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	pool, err := postgres.NewPool(ctx, "postgres://postgres:postgres@localhost:54348/postgres?sslmode=disable")
+	pool, err := postgres.NewPool(ctx, "postgres://postgres:postgres@localhost:15448/postgres?sslmode=disable")
 	if err != nil {
 		t.Fatalf("connect pool: %v", err)
 	}

services/protocols/internal/e2e/f2_complete_e2e_test.go

@@ -35,7 +35,7 @@ func TestF2CompleteSuiteEndToEnd(t *testing.T) {
 	// 1. Embedded Postgres Setup
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54335).
+		Port(15435).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -43,7 +43,7 @@ func TestF2CompleteSuiteEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54335/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15435/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect pool: %v", err)

services/protocols/internal/e2e/f3_complete_e2e_test.go

@@ -80,7 +80,7 @@ func TestPhase3CompleteEndToEnd(t *testing.T) {
 	// Initialize embedded PostgreSQL
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54340).
+		Port(15440).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -88,7 +88,7 @@ func TestPhase3CompleteEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54340/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15440/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect pool: %v", err)

services/protocols/internal/e2e/http_services_e2e_test.go

@@ -25,7 +25,7 @@ func TestHttpServicesEndToEnd(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54337).
+		Port(15437).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -33,7 +33,7 @@ func TestHttpServicesEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54337/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15437/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect pool: %v", err)

services/protocols/internal/e2e/imap_core_e2e_test.go

@@ -30,7 +30,7 @@ func TestImapCoreEndToEnd(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54330). // distinct from the SMTP E2E test's 54329, so both can run concurrently
+		Port(15430). // distinct from the SMTP E2E test's 15429, so both can run concurrently
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -38,7 +38,7 @@ func TestImapCoreEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54330/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15430/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect: %v", err)

services/protocols/internal/e2e/imap_idle_condstore_e2e_test.go

@@ -30,7 +30,7 @@ func TestImapIdleAndCondStoreEndToEnd(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54331). // distinct port for concurrent e2e test execution
+		Port(15431). // distinct port for concurrent e2e test execution
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -38,7 +38,7 @@ func TestImapIdleAndCondStoreEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54331/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15431/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect: %v", err)

services/protocols/internal/e2e/managesieve_e2e_test.go

@@ -28,7 +28,7 @@ func TestManageSieveEndToEnd(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54333). // distinct port for ManageSieve e2e test execution
+		Port(15433). // distinct port for ManageSieve e2e test execution
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -36,7 +36,7 @@ func TestManageSieveEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54333/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15433/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect: %v", err)

services/protocols/internal/e2e/outbound_queue_e2e_test.go

@@ -33,7 +33,7 @@ func TestOutboundQueueEndToEnd(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54339).
+		Port(15439).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -41,7 +41,7 @@ func TestOutboundQueueEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54339/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15439/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect pool: %v", err)

services/protocols/internal/e2e/pop3_e2e_test.go

@@ -29,7 +29,7 @@ func TestPop3EndToEnd(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54332). // distinct port for POP3 e2e test execution
+		Port(15432). // distinct port for POP3 e2e test execution
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -37,7 +37,7 @@ func TestPop3EndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54332/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15432/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect: %v", err)

services/protocols/internal/e2e/ports_test.go

@@ -0,0 +1,107 @@
+package e2e
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// Every embedded Postgres in this package binds a fixed port, and those ports
+// must sit below the kernel's ephemeral range.
+//
+// They did not. All sixteen were inside 32768-60999, which is the range Linux
+// draws source ports from for outgoing connections. Nothing in this repository
+// was listening on them - the kernel simply handed one out to some unrelated
+// connection at the moment a test tried to bind it, and that test died with
+// "process already listening on port 54340". It is a race, so it failed rarely
+// and never in the same place twice, which is exactly what makes it expensive.
+//
+// This test reads the ports out of the source rather than from a list someone
+// has to remember to update: the next person adding an e2e test gets told here
+// instead of by a red build a week later.
+
+// ephemeralFloor is the lowest port Linux will hand out as a source port
+// (net.ipv4.ip_local_port_range). Staying below it means the kernel will never
+// allocate one of these behind our back.
+const ephemeralFloor = 32768
+
+var portPattern = regexp.MustCompile(`Port\((\d+)\)`)
+
+func TestEmbeddedPostgresPortsAreOutsideTheEphemeralRange(t *testing.T) {
+	entries, err := filepath.Glob("*_test.go")
+	if err != nil {
+		t.Fatalf("glob: %v", err)
+	}
+
+	found := 0
+	for _, name := range entries {
+		source, err := os.ReadFile(name)
+		if err != nil {
+			t.Fatalf("read %s: %v", name, err)
+		}
+		for _, match := range portPattern.FindAllStringSubmatch(string(source), -1) {
+			port, err := strconv.Atoi(match[1])
+			if err != nil {
+				continue
+			}
+			found++
+			if port >= ephemeralFloor {
+				t.Errorf("%s binds port %d, which is inside the kernel's ephemeral "+
+					"range (>= %d): the kernel can hand it to an unrelated connection "+
+					"and this test will fail intermittently with "+
+					"\"process already listening\". Pick a port below %d.",
+					name, port, ephemeralFloor, ephemeralFloor)
+			}
+		}
+	}
+
+	if found == 0 {
+		t.Fatal("no embedded Postgres ports found; this test is no longer checking anything")
+	}
+}
+
+// Two tests sharing a port would collide every time rather than intermittently.
+func TestEmbeddedPostgresPortsAreUnique(t *testing.T) {
+	entries, _ := filepath.Glob("*_test.go")
+
+	seen := map[int]string{}
+	for _, name := range entries {
+		source, err := os.ReadFile(name)
+		if err != nil {
+			t.Fatalf("read %s: %v", name, err)
+		}
+		for _, match := range portPattern.FindAllStringSubmatch(string(source), -1) {
+			port, _ := strconv.Atoi(match[1])
+			if previous, taken := seen[port]; taken {
+				t.Errorf("port %d is bound by both %s and %s", port, previous, name)
+			}
+			seen[port] = name
+		}
+	}
+}
+
+// The port in the connection string has to be the one the server was started
+// on, or the test connects to whatever else happens to be there.
+func TestEachTestConnectsToThePortItStarted(t *testing.T) {
+	entries, _ := filepath.Glob("*_test.go")
+
+	for _, name := range entries {
+		source, err := os.ReadFile(name)
+		if err != nil {
+			t.Fatalf("read %s: %v", name, err)
+		}
+		text := string(source)
+		matches := portPattern.FindAllStringSubmatch(text, -1)
+		if len(matches) != 1 {
+			continue
+		}
+		port := matches[0][1]
+		if strings.Contains(text, "postgres://") && !strings.Contains(text, "localhost:"+port) {
+			t.Errorf("%s starts Postgres on port %s but its connection string points "+
+				"somewhere else", name, port)
+		}
+	}
+}

services/protocols/internal/e2e/smtp_inbound_e2e_test.go

@@ -43,7 +43,7 @@ func TestInboundSMTPEndToEnd(t *testing.T) {
 	// --- 1. Real Postgres via embedded binaries ---------------------------
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54329).
+		Port(15429).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -51,7 +51,7 @@ func TestInboundSMTPEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54329/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15429/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect: %v", err)

services/protocols/internal/e2e/submission_e2e_test.go

@@ -37,7 +37,7 @@ func TestSubmissionEndToEnd(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54344).
+		Port(15444).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -45,7 +45,7 @@ func TestSubmissionEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	dbURL := "postgres://postgres:postgres@localhost:54344/postgres?sslmode=disable"
+	dbURL := "postgres://postgres:postgres@localhost:15444/postgres?sslmode=disable"
 	pool, err := postgres.NewPool(ctx, dbURL)
 	if err != nil {
 		t.Fatalf("connect pool: %v", err)

services/protocols/internal/e2e/webmail_api_e2e_test.go

@@ -58,7 +58,7 @@ func TestWebmailApiEndToEnd(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54346).
+		Port(15446).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -66,7 +66,7 @@ func TestWebmailApiEndToEnd(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	pool, err := postgres.NewPool(ctx, "postgres://postgres:postgres@localhost:54346/postgres?sslmode=disable")
+	pool, err := postgres.NewPool(ctx, "postgres://postgres:postgres@localhost:15446/postgres?sslmode=disable")
 	if err != nil {
 		t.Fatalf("connect pool: %v", err)
 	}
@@ -258,7 +258,7 @@ func TestWebmailApiRejectsCrossMailboxAccess(t *testing.T) {
 
 	runtimeDir := t.TempDir()
 	pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().
-		Port(54347).
+		Port(15447).
 		RuntimePath(runtimeDir).
 		StartTimeout(120 * time.Second))
 	if err := pg.Start(); err != nil {
@@ -266,7 +266,7 @@ func TestWebmailApiRejectsCrossMailboxAccess(t *testing.T) {
 	}
 	defer pg.Stop()
 
-	pool, err := postgres.NewPool(ctx, "postgres://postgres:postgres@localhost:54347/postgres?sslmode=disable")
+	pool, err := postgres.NewPool(ctx, "postgres://postgres:postgres@localhost:15447/postgres?sslmode=disable")
 	if err != nil {
 		t.Fatalf("connect pool: %v", err)
 	}