#19: fix F1-F3 defects found auditing the phases against PLAN.md

Every fix here is a case where the code and the plan disagreed, and the
code was wrong.

Certificates (section 8.2, 8.4). The parser documented that it skipped
expired entries but never checked, so an expired certificate would be
served until every handshake failed. LastReload() only advanced when
acme.json changed - and it does not change for sixty days - which made
the timestamp risk R2 monitors indistinguishable from a watcher gone
blind. Expiry is now checked on every pass, LastReload records the check
and LastChange the swap, and the <7d/<24h alerts of section 8.4 exist.

ARC (section 5). buildArcSealer resolved the DKIM key at startup, before
runDnsReconciler had provisioned one. On a first boot ARC was disabled
until someone restarted the process, which the log even admitted while
offering no way for it to start. A LazySealer binds on first use instead.

Health (section 8.4). There was no degraded state at all. /health now
reports one - still 200, because restarting cannot fix a certificate
Traefik never issued.

Bounces. DSNs were sent From postmaster@lambdamail.local, a domain that
does not resolve, so the notification was itself undeliverable. They now
carry the configured mail host, a Message-ID, and Auto-Submitted so a
vacation responder cannot loop against them.

Fan-out atomicity (section 6.1). Persist ran one transaction per
recipient, so a failure halfway through an alias fan-out left earlier
recipients with a copy while the sender was told to retry - and the retry
delivered to them twice. PersistAll commits the whole delivery or none of
it. Verified against Postgres by reverting the repository to the old
behaviour and confirming the new test fails on it.

SMTP replies. The use case encoded replies in error text ("550 5.7.1 ...")
which the adapter re-parsed by prefix, so rewording a message could turn a
permanent refusal into a retry. SmtpRejection carries the decision and the
adapter renders it.

Repository tests. These only run with TEST_DATABASE_URL, which CI does
set - so the job was red: copy and expunge seeded a 68-character digest
into a CHAR(64) column, and three others used fixed digests that collide
with the UNIQUE index on any re-run. A shared testBlobDigest() helper
fixes both; the suite now passes twice in a row against one database.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

services/protocols/internal/infrastructure/arc/lazy_sealer.go

@@ -0,0 +1,75 @@
+package arc
+
+import (
+	"context"
+	"crypto"
+	"sync"
+
+	"lambdamail/protocols/internal/application/port"
+)
+
+// KeyParser turns a stored private key into a signer. It is injected so this
+// package stays independent of how DKIM keys are encoded.
+type KeyParser func(privateKeyPEM []byte) (crypto.Signer, error)
+
+// LazySealer resolves the signing key on first use instead of at construction.
+//
+// The composition root builds the sealer before the DKIM provisioner has run,
+// so on a first boot no key exists yet. Binding to that empty result would
+// disable ARC until the process was restarted by hand - the failure PLAN.md
+// section 5 cannot tolerate, because the whole point of ARC is that it is
+// always applied, not applied when the ordering happened to work out.
+type LazySealer struct {
+	keys       port.DkimKeyRepository
+	parse      KeyParser
+	domain     string
+	authServID string
+
+	mu       sync.Mutex
+	resolved port.ArcSealer
+}
+
+// NewLazySealer returns a sealer that binds to the domain's active DKIM key
+// the first time a message needs sealing.
+func NewLazySealer(keys port.DkimKeyRepository, parse KeyParser, domain, authServID string) *LazySealer {
+	return &LazySealer{keys: keys, parse: parse, domain: domain, authServID: authServID}
+}
+
+// Seal delegates to the underlying sealer, resolving it if needed. Until a key
+// exists the message is returned untouched: an unsealed message is delivered,
+// which is strictly better than a rejected one.
+func (l *LazySealer) Seal(ctx context.Context, message []byte, authResult port.InboundAuthResult) ([]byte, error) {
+	sealer, err := l.sealer(ctx)
+	if err != nil || sealer == nil {
+		return message, err
+	}
+	return sealer.Seal(ctx, message, authResult)
+}
+
+func (l *LazySealer) sealer(ctx context.Context) (port.ArcSealer, error) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	if l.resolved != nil {
+		return l.resolved, nil
+	}
+
+	active, err := l.keys.FindActiveKeys(ctx, l.domain)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, key := range active {
+		signer, err := l.parse(key.PrivateKeyPEM)
+		if err != nil {
+			continue
+		}
+		l.resolved = NewSealer(l.domain, key.Selector, l.authServID, signer)
+		return l.resolved, nil
+	}
+
+	// Still no usable key. Leaving resolved nil means the next message tries
+	// again, which is what turns "not provisioned yet" into a delay rather
+	// than a permanent loss of sealing.
+	return nil, nil
+}

services/protocols/internal/infrastructure/arc/lazy_sealer_test.go

@@ -0,0 +1,108 @@
+package arc
+
+import (
+	"context"
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"strings"
+	"sync"
+	"testing"
+
+	"lambdamail/protocols/internal/application/port"
+)
+
+// stubKeyRepo serves whatever the test decides is provisioned at that moment.
+type stubKeyRepo struct {
+	mu   sync.Mutex
+	keys []port.DkimSigningKey
+	// calls counts lookups, to prove the resolved key is cached.
+	calls int
+}
+
+func (s *stubKeyRepo) FindActiveKeys(_ context.Context, _ string) ([]port.DkimSigningKey, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.calls++
+	return s.keys, nil
+}
+
+func (s *stubKeyRepo) provision(t *testing.T) {
+	t.Helper()
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("generate key: %v", err)
+	}
+	pemBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(key),
+	})
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.keys = []port.DkimSigningKey{{
+		DomainName:    "example.test",
+		Selector:      "lmail",
+		Algorithm:     "rsa2048",
+		PrivateKeyPEM: pemBytes,
+	}}
+}
+
+func parseTestKey(pemBytes []byte) (crypto.Signer, error) {
+	block, _ := pem.Decode(pemBytes)
+	return x509.ParsePKCS1PrivateKey(block.Bytes)
+}
+
+const testMessage = "From: sender@remote.test\r\n" +
+	"To: user@example.test\r\n" +
+	"Subject: hello\r\n" +
+	"\r\n" +
+	"body\r\n"
+
+// The composition root builds the sealer before the DKIM provisioner has run.
+// Binding at construction would leave ARC off until the next restart.
+func TestLazySealer_StartsSealingOnceAKeyIsProvisioned(t *testing.T) {
+	repo := &stubKeyRepo{}
+	sealer := NewLazySealer(repo, parseTestKey, "example.test", "mail.example.test")
+	ctx := context.Background()
+	authResult := port.InboundAuthResult{SPF: port.AuthResultPass}
+
+	// No key yet: the message goes through unsealed rather than failing.
+	out, err := sealer.Seal(ctx, []byte(testMessage), authResult)
+	if err != nil {
+		t.Fatalf("Seal without a key: %v", err)
+	}
+	if strings.Contains(string(out), "ARC-Seal") {
+		t.Fatal("a message was sealed before any key existed")
+	}
+
+	repo.provision(t)
+
+	out, err = sealer.Seal(ctx, []byte(testMessage), authResult)
+	if err != nil {
+		t.Fatalf("Seal after provisioning: %v", err)
+	}
+	if !strings.Contains(string(out), "ARC-Seal") {
+		t.Fatal("sealing never started after the key was provisioned")
+	}
+}
+
+// Once bound, the key is not looked up again on every message.
+func TestLazySealer_CachesTheResolvedKey(t *testing.T) {
+	repo := &stubKeyRepo{}
+	repo.provision(t)
+	sealer := NewLazySealer(repo, parseTestKey, "example.test", "mail.example.test")
+	ctx := context.Background()
+
+	for range 3 {
+		if _, err := sealer.Seal(ctx, []byte(testMessage), port.InboundAuthResult{}); err != nil {
+			t.Fatalf("Seal: %v", err)
+		}
+	}
+
+	if repo.calls != 1 {
+		t.Errorf("FindActiveKeys called %d times, want 1", repo.calls)
+	}
+}

services/protocols/internal/infrastructure/postgres/copy_repository_test.go

@@ -42,7 +42,7 @@ func TestCopyRepository_CopyMessages_AllocatesNewUIDsAndPreservesFlags(t *testin
 	}
 	blobID := uuid.New()
 	_, err = pool.Exec(ctx, `INSERT INTO message_blobs (id, content_sha256, storage_driver, storage_path, size_bytes, ref_count) VALUES ($1, $2, 'local', '/tmp/x', 10, 1)`,
-		blobID, uuid.New().String()+uuid.New().String()[:32])
+		blobID, testBlobDigest())
 	if err != nil {
 		t.Fatalf("seed blob: %v", err)
 	}

services/protocols/internal/infrastructure/postgres/expunge_repository_test.go

@@ -35,7 +35,7 @@ func TestExpungeRepository_Expunge_SetsExpungedAtAndDecrementsCounters(t *testin
 	}
 	blobID := uuid.New()
 	_, err = pool.Exec(ctx, `INSERT INTO message_blobs (id, content_sha256, storage_driver, storage_path, size_bytes) VALUES ($1, $2, 'local', '/tmp/x', 10)`,
-		blobID, uuid.New().String()+uuid.New().String()[:32])
+		blobID, testBlobDigest())
 	if err != nil {
 		t.Fatalf("seed blob: %v", err)
 	}

services/protocols/internal/infrastructure/postgres/flag_repository_test.go

@@ -37,7 +37,7 @@ func TestFlagRepository_SetFlags_AddThenDelThenSet(t *testing.T) {
 	}
 	blobID := uuid.New()
 	_, err = pool.Exec(ctx, `INSERT INTO message_blobs (id, content_sha256, storage_driver, storage_path, size_bytes) VALUES ($1, $2, 'local', '/tmp/x', 10)`,
-		blobID, "1111110000000000000000000000000000000000000000000000000000000000"[:64])
+		blobID, testBlobDigest())
 	if err != nil {
 		t.Fatalf("seed blob: %v", err)
 	}

services/protocols/internal/infrastructure/postgres/inbound_message_repository.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgxpool"
 
 	"lambdamail/protocols/internal/application/port"
@@ -21,13 +22,37 @@ func NewInboundMessageRepository(pool *pgxpool.Pool) *InboundMessageRepository {
 	return &InboundMessageRepository{pool: pool}
 }
 
-func (r *InboundMessageRepository) Persist(ctx context.Context, input port.PersistInboundMessageInput) (int64, error) {
+// PersistAll records every recipient's copy in one transaction, so a delivery
+// that fans out is all-or-nothing and a retried SMTP transaction cannot leave
+// an earlier recipient with a duplicate.
+func (r *InboundMessageRepository) PersistAll(ctx context.Context, inputs []port.PersistInboundMessageInput) ([]int64, error) {
+	if len(inputs) == 0 {
+		return nil, nil
+	}
+
 	tx, err := r.pool.Begin(ctx)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
 	defer tx.Rollback(ctx) // no-op if Commit succeeds
 
+	uids := make([]int64, 0, len(inputs))
+	for _, input := range inputs {
+		uid, err := persistOne(ctx, tx, input)
+		if err != nil {
+			return nil, err
+		}
+		uids = append(uids, uid)
+	}
+
+	if err := tx.Commit(ctx); err != nil {
+		return nil, fmt.Errorf("commit: %w", err)
+	}
+	return uids, nil
+}
+
+// persistOne writes one recipient's copy inside the caller's transaction.
+func persistOne(ctx context.Context, tx pgx.Tx, input port.PersistInboundMessageInput) (int64, error) {
 	targetFolder := input.TargetFolderName
 	if targetFolder == "" {
 		targetFolder = "INBOX"
@@ -35,7 +60,7 @@ func (r *InboundMessageRepository) Persist(ctx context.Context, input port.Persi
 
 	var folderID string
 	var uid int64
-	err = tx.QueryRow(ctx, `
+	err := tx.QueryRow(ctx, `
 		SELECT id, uid_next FROM folders
 		WHERE mailbox_id = $1 AND (special_use = LOWER($2) OR LOWER(name) = LOWER($2))
 		ORDER BY special_use IS NOT NULL DESC
@@ -84,8 +109,5 @@ func (r *InboundMessageRepository) Persist(ctx context.Context, input port.Persi
 		return 0, fmt.Errorf("insert outbox event: %w", err)
 	}
 
-	if err := tx.Commit(ctx); err != nil {
-		return 0, fmt.Errorf("commit: %w", err)
-	}
 	return uid, nil
 }

services/protocols/internal/infrastructure/postgres/inbound_message_repository_test.go

@@ -38,7 +38,7 @@ func TestInboundMessageRepository_Persist_AllocatesSequentialUIDs(t *testing.T)
 
 	blobID := uuid.New()
 	_, err = pool.Exec(ctx, `INSERT INTO message_blobs (id, content_sha256, storage_driver, storage_path, size_bytes) VALUES ($1, $2, 'local', '/tmp/x', 10)`,
-		blobID, "deadbeef00000000000000000000000000000000000000000000000000000000"[:64])
+		blobID, testBlobDigest())
 	if err != nil {
 		t.Fatalf("seed blob: %v", err)
 	}
@@ -46,22 +46,23 @@ func TestInboundMessageRepository_Persist_AllocatesSequentialUIDs(t *testing.T)
 	repo := NewInboundMessageRepository(pool)
 	blob := port.BlobRef{ID: blobID, SHA256: "deadbeef", SizeBytes: 10}
 
-	firstUID, err := repo.Persist(ctx, port.PersistInboundMessageInput{
+	firstUIDs, err := repo.PersistAll(ctx, []port.PersistInboundMessageInput{{
 		MailboxID: mailboxID, Blob: blob,
 		SenderAddress: "sender@example.test", RecipientAddress: "user@msgrepo-test-" + domainID.String() + ".invalid",
 		SPFResult: "none", DKIMResult: "none", DMARCResult: "none",
-	})
+	}})
 	if err != nil {
-		t.Fatalf("first Persist: %v", err)
+		t.Fatalf("first PersistAll: %v", err)
 	}
-	secondUID, err := repo.Persist(ctx, port.PersistInboundMessageInput{
+	secondUIDs, err := repo.PersistAll(ctx, []port.PersistInboundMessageInput{{
 		MailboxID: mailboxID, Blob: blob,
 		SenderAddress: "sender2@example.test", RecipientAddress: "user@msgrepo-test-" + domainID.String() + ".invalid",
 		SPFResult: "none", DKIMResult: "none", DMARCResult: "none",
-	})
+	}})
 	if err != nil {
-		t.Fatalf("second Persist: %v", err)
+		t.Fatalf("second PersistAll: %v", err)
 	}
+	firstUID, secondUID := firstUIDs[0], secondUIDs[0]
 	if secondUID <= firstUID {
 		t.Errorf("secondUID (%d) must be strictly greater than firstUID (%d) - RFC 3501 section 2.3.1.1", secondUID, firstUID)
 	}
@@ -71,7 +72,7 @@ func TestInboundMessageRepository_Persist_AllocatesSequentialUIDs(t *testing.T)
 		t.Fatalf("query ref_count: %v", err)
 	}
 	if refCount != 2 {
-		t.Errorf("ref_count = %d, want 2 after two Persist calls referencing the same blob", refCount)
+		t.Errorf("ref_count = %d, want 2 after two PersistAll calls referencing the same blob", refCount)
 	}
 
 	var unreadCount, totalCount int
@@ -79,10 +80,10 @@ func TestInboundMessageRepository_Persist_AllocatesSequentialUIDs(t *testing.T)
 		t.Fatalf("query folder counters: %v", err)
 	}
 	if unreadCount != 2 {
-		t.Errorf("folder unread_count = %d, want 2 after two Persist calls", unreadCount)
+		t.Errorf("folder unread_count = %d, want 2 after two PersistAll calls", unreadCount)
 	}
 	if totalCount != 2 {
-		t.Errorf("folder total_count = %d, want 2 after two Persist calls", totalCount)
+		t.Errorf("folder total_count = %d, want 2 after two PersistAll calls", totalCount)
 	}
 
 	var usedBytes int64
@@ -90,7 +91,7 @@ func TestInboundMessageRepository_Persist_AllocatesSequentialUIDs(t *testing.T)
 		t.Fatalf("query mailbox used_bytes: %v", err)
 	}
 	if usedBytes != blob.SizeBytes*2 {
-		t.Errorf("mailbox used_bytes = %d, want %d (two Persist calls of %d bytes each)", usedBytes, blob.SizeBytes*2, blob.SizeBytes)
+		t.Errorf("mailbox used_bytes = %d, want %d (two PersistAll calls of %d bytes each)", usedBytes, blob.SizeBytes*2, blob.SizeBytes)
 	}
 }
 
@@ -118,18 +119,88 @@ func TestInboundMessageRepository_Persist_ReturnsErrorWhenNoInboxFolderExists(t
 
 	blobID := uuid.New()
 	_, err = pool.Exec(ctx, `INSERT INTO message_blobs (id, content_sha256, storage_driver, storage_path, size_bytes) VALUES ($1, $2, 'local', '/tmp/x', 10)`,
-		blobID, "cafebabe00000000000000000000000000000000000000000000000000000000"[:64])
+		blobID, testBlobDigest())
 	if err != nil {
 		t.Fatalf("seed blob: %v", err)
 	}
 
 	repo := NewInboundMessageRepository(pool)
-	_, err = repo.Persist(ctx, port.PersistInboundMessageInput{
+	_, err = repo.PersistAll(ctx, []port.PersistInboundMessageInput{{
 		MailboxID: mailboxID, Blob: port.BlobRef{ID: blobID, SizeBytes: 10},
 		SenderAddress: "sender@example.test", RecipientAddress: "nofolder@example.test",
 		SPFResult: "none", DKIMResult: "none", DMARCResult: "none",
-	})
+	}})
 	if err == nil {
 		t.Fatal("expected an error when the mailbox has no INBOX folder, got nil")
 	}
 }
+
+// An alias fanning out is one SMTP transaction with one reply. If a later
+// recipient's insert fails, the earlier ones must not remain: the sender is
+// told to retry, and the retry would deliver to them a second time.
+func TestInboundMessageRepository_PersistAll_RollsBackEveryRecipientOnFailure(t *testing.T) {
+	pool := testPool(t)
+	defer pool.Close()
+	ctx := context.Background()
+
+	domainID := uuid.New()
+	domainName := "fanout-test-" + domainID.String() + ".invalid"
+	if _, err := pool.Exec(ctx, `INSERT INTO domains (id, name, punycode_name) VALUES ($1, $2, $3)`,
+		domainID, domainName, domainName); err != nil {
+		t.Fatalf("seed domain: %v", err)
+	}
+	defer pool.Exec(ctx, `DELETE FROM domains WHERE id = $1`, domainID)
+
+	// The first mailbox has an INBOX, the second deliberately does not, so
+	// persisting the pair fails on the second entry.
+	goodMailboxID, badMailboxID := uuid.New(), uuid.New()
+	for i, id := range []uuid.UUID{goodMailboxID, badMailboxID} {
+		local := []string{"good", "bad"}[i]
+		if _, err := pool.Exec(ctx, `INSERT INTO mailboxes (id, domain_id, local_part, email_address, password_hash) VALUES ($1, $2, $3, $4, '$argon2id$v=19$m=65536,t=3,p=4$c29tZXNhbHQ$RdescudvJCsgt3ub+b+dWRWJTmaaJObG')`,
+			id, domainID, local, local+"@"+domainName); err != nil {
+			t.Fatalf("seed mailbox %s: %v", local, err)
+		}
+	}
+
+	folderID := uuid.New()
+	if _, err := pool.Exec(ctx, `INSERT INTO folders (id, mailbox_id, name, special_use) VALUES ($1, $2, 'INBOX', 'inbox')`,
+		folderID, goodMailboxID); err != nil {
+		t.Fatalf("seed folder: %v", err)
+	}
+
+	blobID := uuid.New()
+	blobDigest := testBlobDigest()
+	if _, err := pool.Exec(ctx, `INSERT INTO message_blobs (id, content_sha256, storage_driver, storage_path, size_bytes) VALUES ($1, $2, 'local', '/tmp/x', 10)`,
+		blobID, blobDigest); err != nil {
+		t.Fatalf("seed blob: %v", err)
+	}
+	defer pool.Exec(ctx, `DELETE FROM message_blobs WHERE id = $1`, blobID)
+	blob := port.BlobRef{ID: blobID, SHA256: blobDigest, SizeBytes: 10}
+
+	repo := NewInboundMessageRepository(pool)
+	_, err := repo.PersistAll(ctx, []port.PersistInboundMessageInput{
+		{MailboxID: goodMailboxID, Blob: blob, SenderAddress: "sender@remote.test", RecipientAddress: "alias@" + domainName,
+			SPFResult: "none", DKIMResult: "none", DMARCResult: "none"},
+		{MailboxID: badMailboxID, Blob: blob, SenderAddress: "sender@remote.test", RecipientAddress: "alias@" + domainName,
+			SPFResult: "none", DKIMResult: "none", DMARCResult: "none"},
+	})
+	if err == nil {
+		t.Fatal("PersistAll succeeded even though the second recipient has no INBOX")
+	}
+
+	var messageCount int
+	if err := pool.QueryRow(ctx, `SELECT count(*) FROM email_messages WHERE mailbox_id = $1`, goodMailboxID).Scan(&messageCount); err != nil {
+		t.Fatalf("count messages: %v", err)
+	}
+	if messageCount != 0 {
+		t.Errorf("the first recipient kept %d message(s) after a failed fan-out; the sender's retry would duplicate them", messageCount)
+	}
+
+	var refCount int
+	if err := pool.QueryRow(ctx, `SELECT ref_count FROM message_blobs WHERE id = $1`, blobID).Scan(&refCount); err != nil {
+		t.Fatalf("query ref_count: %v", err)
+	}
+	if refCount != 0 {
+		t.Errorf("ref_count = %d after a rolled-back delivery, want 0", refCount)
+	}
+}

services/protocols/internal/infrastructure/postgres/mailbox_repository_test.go

@@ -3,6 +3,7 @@ package postgres
 import (
 	"context"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/google/uuid"
@@ -309,3 +310,10 @@ func TestMailboxRepository_ResolveDeliveryTargets_SkipsAliasDestinationThatIsNot
 		t.Errorf("targets = %+v, want empty slice - external forwarding destinations are not internal mailboxes and outbound delivery does not exist yet", targets)
 	}
 }
+
+// testBlobDigest produces a unique 64-character value for message_blobs'
+// content_sha256, which is UNIQUE in the schema. A hardcoded digest makes the
+// test pass exactly once per database and fail on every re-run.
+func testBlobDigest() string {
+	return strings.ReplaceAll(uuid.New().String()+uuid.New().String(), "-", "")[:64]
+}

services/protocols/internal/infrastructure/postgres/message_query_repository_test.go

@@ -35,7 +35,7 @@ func TestMessageQueryRepository_ListMessages_ReturnsRowsOrderedByUID(t *testing.
 	}
 	blobID := uuid.New()
 	_, err = pool.Exec(ctx, `INSERT INTO message_blobs (id, content_sha256, storage_driver, storage_path, size_bytes) VALUES ($1, $2, 'local', '/tmp/x', 42)`,
-		blobID, "abcdef0000000000000000000000000000000000000000000000000000000000"[:64])
+		blobID, testBlobDigest())
 	if err != nil {
 		t.Fatalf("seed blob: %v", err)
 	}

services/protocols/internal/infrastructure/tls/acme_cert_watcher.go

@@ -3,6 +3,7 @@ package tlsprovider
 import (
 	"crypto/sha256"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -37,8 +38,15 @@ type AcmeCertWatcher struct {
 	// using a consistent map.
 	certificates atomic.Pointer[map[string]*tls.Certificate]
 
-	lastHash     atomic.Pointer[[32]byte]
+	lastHash atomic.Pointer[[32]byte]
+	// lastReloadAt is the last time the store was successfully read and found
+	// usable, whether or not its content had changed. It is what PLAN.md risk
+	// R2 monitors: acme.json normally sits untouched for sixty days, so a
+	// timestamp that only advanced on change could not distinguish a healthy
+	// idle watcher from one that has gone blind.
 	lastReloadAt atomic.Pointer[time.Time]
+	// lastChangeAt is the last time the certificates themselves were swapped.
+	lastChangeAt atomic.Pointer[time.Time]
 
 	// unknownSNI counts handshakes for names we hold no certificate for; it
 	// backs the tls_unknown_sni_total metric of PLAN.md section 8.4.
@@ -110,31 +118,80 @@ func (w *AcmeCertWatcher) reload() error {
 		return fmt.Errorf("read %s: %w", w.path, err)
 	}
 
-	hash := sha256.Sum256(raw)
-	if previous := w.lastHash.Load(); previous != nil && *previous == hash {
-		return nil
-	}
-
-	certificates, err := parseAcmeStore(raw)
-	if err != nil {
-		return err
-	}
-	if len(certificates) == 0 {
-		return fmt.Errorf("acme watcher: %s holds no usable certificate", w.path)
-	}
-
-	w.certificates.Store(&certificates)
-	w.lastHash.Store(&hash)
 	now := time.Now()
-	w.lastReloadAt.Store(&now)
 
+	hash := sha256.Sum256(raw)
+	if previous := w.lastHash.Load(); previous == nil || *previous != hash {
+		certificates, err := parseAcmeStore(raw, now)
+		if err != nil {
+			return err
+		}
+		if len(certificates) == 0 {
+			return fmt.Errorf("acme watcher: %s holds no usable certificate", w.path)
+		}
+
+		w.certificates.Store(&certificates)
+		w.lastHash.Store(&hash)
+		w.lastChangeAt.Store(&now)
+	}
+
+	// Expiry is checked on every pass, not only on change: a certificate ages
+	// out while the file it lives in stays byte-identical.
+	w.reportExpiry(now)
+
+	w.lastReloadAt.Store(&now)
 	return nil
 }
 
+// reportExpiry raises the alerts of PLAN.md section 8.4 for certificates that
+// are close to expiring. Renewal is Traefik's job, so all this side can do is
+// make the failure loud before it becomes an outage.
+func (w *AcmeCertWatcher) reportExpiry(now time.Time) {
+	host, expiry, ok := w.EarliestExpiry()
+	if !ok {
+		return
+	}
+
+	switch remaining := expiry.Sub(now); {
+	case remaining <= 0:
+		log.Printf("CRITICAL: the certificate for %s expired at %s and Traefik has not replaced it", host, expiry.Format(time.RFC3339))
+	case remaining < 24*time.Hour:
+		log.Printf("CRITICAL: the certificate for %s expires in %s", host, remaining.Round(time.Minute))
+	case remaining < 7*24*time.Hour:
+		log.Printf("WARNING: the certificate for %s expires in %s", host, remaining.Round(time.Hour))
+	}
+}
+
+// EarliestExpiry reports the soonest expiry across the loaded certificates,
+// which is the one that decides when the deployment breaks.
+func (w *AcmeCertWatcher) EarliestExpiry() (string, time.Time, bool) {
+	loaded := w.certificates.Load()
+	if loaded == nil {
+		return "", time.Time{}, false
+	}
+
+	var (
+		soonestHost string
+		soonest     time.Time
+	)
+	for host, cert := range *loaded {
+		if cert.Leaf == nil {
+			continue
+		}
+		if soonest.IsZero() || cert.Leaf.NotAfter.Before(soonest) {
+			soonest, soonestHost = cert.Leaf.NotAfter, host
+		}
+	}
+	if soonest.IsZero() {
+		return "", time.Time{}, false
+	}
+	return soonestHost, soonest, true
+}
+
 // parseAcmeStore walks every resolver in the file. A single unusable entry -
 // an expired certificate, or one whose key does not match - is skipped rather
 // than failing the whole load, so one stale domain cannot take the server down.
-func parseAcmeStore(raw []byte) (map[string]*tls.Certificate, error) {
+func parseAcmeStore(raw []byte, now time.Time) (map[string]*tls.Certificate, error) {
 	var store acmeFile
 	if err := json.Unmarshal(raw, &store); err != nil {
 		return nil, fmt.Errorf("parse acme store: %w", err)
@@ -161,6 +218,23 @@ func parseAcmeStore(raw []byte) (map[string]*tls.Certificate, error) {
 				continue
 			}
 
+			// X509KeyPair leaves Leaf nil. Parsing it here is what makes the
+			// expiry checks below and in reportExpiry possible at all.
+			leaf, err := x509.ParseCertificate(pair.Certificate[0])
+			if err != nil {
+				log.Printf("acme watcher: skipping %s in resolver %q: leaf certificate is unparseable: %v", entry.Domain.Main, resolverName, err)
+				continue
+			}
+			// An expired entry is worse than a missing one: it would be served
+			// in a handshake every peer then rejects. Skipping it lets the SNI
+			// fallback or the self-signed provider take over instead
+			// (PLAN.md section 8.2, trap 3).
+			if now.After(leaf.NotAfter) {
+				log.Printf("acme watcher: skipping %s in resolver %q: the certificate expired at %s", entry.Domain.Main, resolverName, leaf.NotAfter.Format(time.RFC3339))
+				continue
+			}
+			pair.Leaf = leaf
+
 			for _, name := range append([]string{entry.Domain.Main}, entry.Domain.SANs...) {
 				if name != "" {
 					certificates[strings.ToLower(name)] = &pair
@@ -210,9 +284,9 @@ func wildcardOf(name string) string {
 	return ""
 }
 
-// LastReload reports when the store was last successfully refreshed. A
-// timestamp that stops advancing is the signal that the watcher went blind
-// (PLAN.md risk R2).
+// LastReload reports when the store was last read successfully, changed or
+// not. A timestamp that stops advancing is the signal that the watcher went
+// blind (PLAN.md risk R2).
 func (w *AcmeCertWatcher) LastReload() time.Time {
 	if t := w.lastReloadAt.Load(); t != nil {
 		return *t
@@ -220,6 +294,15 @@ func (w *AcmeCertWatcher) LastReload() time.Time {
 	return time.Time{}
 }
 
+// LastChange reports when the served certificates were last swapped, which is
+// how a renewal that never arrived is detected.
+func (w *AcmeCertWatcher) LastChange() time.Time {
+	if t := w.lastChangeAt.Load(); t != nil {
+		return *t
+	}
+	return time.Time{}
+}
+
 // UnknownSNICount backs the tls_unknown_sni_total metric.
 func (w *AcmeCertWatcher) UnknownSNICount() int64 {
 	return w.unknownSNI.Load()

services/protocols/internal/infrastructure/tls/acme_cert_watcher_test.go

@@ -20,6 +20,13 @@ import (
 // makeCertPEM issues a self-signed certificate for the given names.
 func makeCertPEM(t *testing.T, names ...string) (certPEM, keyPEM []byte) {
 	t.Helper()
+	return makeCertPEMWithValidity(t, -time.Hour, 90*24*time.Hour, names...)
+}
+
+// makeCertPEMWithValidity issues a certificate whose validity window is given
+// relative to now, so a test can produce an expired or nearly expired one.
+func makeCertPEMWithValidity(t *testing.T, notBefore, notAfter time.Duration, names ...string) (certPEM, keyPEM []byte) {
+	t.Helper()
 
 	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {
@@ -29,8 +36,8 @@ func makeCertPEM(t *testing.T, names ...string) (certPEM, keyPEM []byte) {
 		SerialNumber: big.NewInt(time.Now().UnixNano()),
 		Subject:      pkix.Name{CommonName: names[0]},
 		DNSNames:     names,
-		NotBefore:    time.Now().Add(-time.Hour),
-		NotAfter:     time.Now().Add(90 * 24 * time.Hour),
+		NotBefore:    time.Now().Add(notBefore),
+		NotAfter:     time.Now().Add(notAfter),
 	}
 	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
 	if err != nil {
@@ -49,6 +56,13 @@ func makeCertPEM(t *testing.T, names ...string) (certPEM, keyPEM []byte) {
 // resolver name to that resolver's account and certificates.
 func writeAcmeStore(t *testing.T, dir, resolverName string, entries map[string][]string) string {
 	t.Helper()
+	return writeAcmeStoreWithValidity(t, dir, resolverName, entries, -time.Hour, 90*24*time.Hour)
+}
+
+// writeAcmeStoreWithValidity is writeAcmeStore with control over how long the
+// issued certificates are valid for.
+func writeAcmeStoreWithValidity(t *testing.T, dir, resolverName string, entries map[string][]string, notBefore, notAfter time.Duration) string {
+	t.Helper()
 
 	type domain struct {
 		Main string   `json:"main"`
@@ -62,7 +76,7 @@ func writeAcmeStore(t *testing.T, dir, resolverName string, entries map[string][
 
 	var certificates []certificate
 	for main, sans := range entries {
-		certPEM, keyPEM := makeCertPEM(t, append([]string{main}, sans...)...)
+		certPEM, keyPEM := makeCertPEMWithValidity(t, notBefore, notAfter, append([]string{main}, sans...)...)
 		certificates = append(certificates, certificate{
 			Domain:      domain{Main: main, SANs: sans},
 			Certificate: base64.StdEncoding.EncodeToString(certPEM),
@@ -240,21 +254,63 @@ func TestAcmeCertWatcher_FailsWhenStoreMissing(t *testing.T) {
 	}
 }
 
-// An identical file must not cause a pointless swap; LastReload only advances
-// when the content actually changed.
-func TestAcmeCertWatcher_SkipsReloadWhenContentUnchanged(t *testing.T) {
+// An identical file must not cause a pointless swap, but the watcher must
+// still record that it looked. acme.json sits untouched for sixty days, so a
+// LastReload that only advanced on change could not tell a healthy idle
+// watcher apart from one that went blind (PLAN.md risk R2).
+func TestAcmeCertWatcher_SkipsSwapButRecordsCheckWhenContentUnchanged(t *testing.T) {
 	dir := t.TempDir()
 	writeAcmeStore(t, dir, "letsencrypt", map[string][]string{"mail.example.test": nil})
 
 	watcher, _ := NewAcmeCertWatcher(dir, "acme.json", "mail.example.test")
-	first := watcher.LastReload()
+	firstReload := watcher.LastReload()
+	firstChange := watcher.LastChange()
 
 	time.Sleep(2 * time.Millisecond)
 	if err := watcher.reload(); err != nil {
 		t.Fatalf("reload: %v", err)
 	}
 
-	if !watcher.LastReload().Equal(first) {
-		t.Error("an unchanged file triggered a reload")
+	if !watcher.LastChange().Equal(firstChange) {
+		t.Error("an unchanged file swapped the certificate store")
+	}
+	if !watcher.LastReload().After(firstReload) {
+		t.Error("LastReload did not advance, so a blind watcher would look identical to an idle one")
+	}
+}
+
+// An expired entry must not be served: every peer would reject the handshake,
+// whereas skipping it lets the SNI fallback take over (PLAN.md section 8.2).
+func TestAcmeCertWatcher_SkipsExpiredCertificate(t *testing.T) {
+	dir := t.TempDir()
+	writeAcmeStoreWithValidity(t, dir, "letsencrypt",
+		map[string][]string{"expired.example.test": nil}, -48*time.Hour, -24*time.Hour)
+
+	if _, err := NewAcmeCertWatcher(dir, "acme.json", "expired.example.test"); err == nil {
+		t.Fatal("a store holding only an expired certificate must not load as usable")
+	}
+}
+
+// The soonest expiry is what decides when the deployment breaks, so it has to
+// be readable for the alerts of PLAN.md section 8.4.
+func TestAcmeCertWatcher_ReportsEarliestExpiry(t *testing.T) {
+	dir := t.TempDir()
+	writeAcmeStoreWithValidity(t, dir, "letsencrypt",
+		map[string][]string{"mail.example.test": nil}, -time.Hour, 72*time.Hour)
+
+	watcher, err := NewAcmeCertWatcher(dir, "acme.json", "mail.example.test")
+	if err != nil {
+		t.Fatalf("NewAcmeCertWatcher: %v", err)
+	}
+
+	host, expiry, ok := watcher.EarliestExpiry()
+	if !ok {
+		t.Fatal("EarliestExpiry reported nothing for a loaded store")
+	}
+	if host != "mail.example.test" {
+		t.Errorf("EarliestExpiry host = %q, want mail.example.test", host)
+	}
+	if remaining := time.Until(expiry); remaining > 73*time.Hour || remaining < 71*time.Hour {
+		t.Errorf("expiry is %s away, want roughly 72h", remaining)
 	}
 }