feat(protocols): conversations - thread_id and in_reply_to written at delivery, list and read APIs

A reply now joins the conversation of the message it answers (Message-ID,
In-Reply-To, References), falling back to the subject for replies with no
headers. /mail/threads lists a folder as conversations and /mail/thread/{id}
returns one. Replying from the webmail flags the answered message.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

services/protocols/internal/application/port/conversations.go

@@ -0,0 +1,68 @@
+package port
+
+import (
+	"context"
+	"time"
+)
+
+// WebmailParticipant is someone who wrote in a conversation.
+type WebmailParticipant struct {
+	Name    string `json:"name"`
+	Address string `json:"address"`
+	// Me marks the mailbox owner, so the list can say "me" instead of a name
+	// the reader knows perfectly well.
+	Me bool `json:"me"`
+}
+
+// WebmailThread is one row of the conversation list: the newest message in the
+// folder being looked at, with what the rest of the conversation adds.
+type WebmailThread struct {
+	ThreadID       string               `json:"thread_id"`
+	UID            uint32               `json:"uid"`
+	Subject        string               `json:"subject"`
+	SenderAddress  string               `json:"sender_address"`
+	FromName       string               `json:"from_display_name"`
+	Snippet        string               `json:"snippet"`
+	ReceivedAt     time.Time            `json:"received_at"`
+	HasAttachments bool                 `json:"has_attachments"`
+	MessageCount   int                  `json:"message_count"`
+	UnseenCount    int                  `json:"unseen_count"`
+	Seen           bool                 `json:"seen"`
+	Flagged        bool                 `json:"flagged"`
+	Answered       bool                 `json:"answered"`
+	Participants   []WebmailParticipant `json:"participants"`
+}
+
+// WebmailThreadMessage is one message inside a conversation.
+type WebmailThreadMessage struct {
+	UID            uint32    `json:"uid"`
+	Folder         string    `json:"folder"`
+	Subject        string    `json:"subject"`
+	SenderAddress  string    `json:"sender_address"`
+	FromName       string    `json:"from_display_name"`
+	Snippet        string    `json:"snippet"`
+	ReceivedAt     time.Time `json:"received_at"`
+	HasAttachments bool      `json:"has_attachments"`
+	Seen           bool      `json:"seen"`
+	Flagged        bool      `json:"flagged"`
+	// Answered is true when the mailbox owner replied, whether the \Answered
+	// flag was set or the reply is simply there in the conversation - a reply
+	// written in another mail program leaves no flag, only the message.
+	Answered bool `json:"answered"`
+	// Own marks a message the mailbox owner wrote.
+	Own bool `json:"own"`
+}
+
+// ConversationReader serves the conversation views.
+type ConversationReader interface {
+	// ListThreads returns the conversations that have a message in the folder,
+	// newest first by that message.
+	ListThreads(ctx context.Context, mailboxID, folderName, search string, limit, offset int) ([]WebmailThread, error)
+	// ThreadMessages returns every message of one conversation, oldest first.
+	// folderName is the folder being viewed: a conversation's messages in Trash
+	// and Junk are left out unless that is the folder.
+	ThreadMessages(ctx context.Context, mailboxID, folderName, threadID string) ([]WebmailThreadMessage, error)
+	// MarkAnsweredByMessageID sets \Answered on the message a reply was written
+	// to, wherever in the mailbox it is filed.
+	MarkAnsweredByMessageID(ctx context.Context, mailboxID, messageID string) error
+}

services/protocols/internal/application/port/inbound_message_repository.go

@@ -22,6 +22,10 @@ type PersistInboundMessageInput struct {
 	FromDisplayName string
 	MessageIDHeader string
 	HasAttachments  bool
+	// InReplyTo and References place the message in its conversation. Both
+	// hold Message-IDs without angle brackets.
+	InReplyTo  string
+	References []string
 	// AlreadySeen marks a copy the user themselves composed - their own Sent
 	// copy, or a draft. It is not unread mail, and counting it as such is why
 	// the Sent folder carried an unread badge that nothing could clear: no

services/protocols/internal/application/usecase/message_headers.go

@@ -19,10 +19,15 @@ type MessageHeaders struct {
 	Subject         string
 	FromDisplayName string
 	// FromAddress is the address in From, lowercased, or empty when it does not parse.
-	FromAddress string
-	MessageID       string
-	Snippet         string
-	HasAttachments  bool
+	FromAddress    string
+	MessageID      string
+	Snippet        string
+	HasAttachments bool
+	// InReplyTo and References are the Message-IDs this message answers, with
+	// the angle brackets removed so they compare equal to MessageID. They are
+	// what ties a reply to the conversation it belongs to.
+	InReplyTo  string
+	References []string
 }
 
 // wordDecoder tolerates the charsets that turn up in real mail. An unknown
@@ -57,6 +62,10 @@ func ExtractMessageHeaders(payload []byte) MessageHeaders {
 
 	out.Subject = truncateUTF8(decodeHeader(msg.Header.Get("Subject")), 998)
 	out.MessageID = truncateUTF8(strings.Trim(msg.Header.Get("Message-ID"), "<>"), 998)
+	if ids := messageIDsIn(msg.Header.Get("In-Reply-To")); len(ids) > 0 {
+		out.InReplyTo = ids[0]
+	}
+	out.References = messageIDsIn(msg.Header.Get("References"))
 
 	if from := msg.Header.Get("From"); from != "" {
 		if addr, err := mail.ParseAddress(from); err == nil {
@@ -152,3 +161,23 @@ func truncateUTF8(s string, max int) string {
 	}
 	return cut
 }
+
+// maxReferences bounds how much of a long chain is kept. The ids nearest the
+// end name the messages closest to this one, which is where its parent is.
+const maxReferences = 20
+
+// messageIDsIn lists the <id> tokens of an In-Reply-To or References header,
+// without their brackets, in the order written.
+func messageIDsIn(header string) []string {
+	var ids []string
+	for _, field := range strings.Fields(header) {
+		id := strings.Trim(field, "<>,;")
+		if id != "" && len(id) <= 998 && strings.Contains(field, "<") {
+			ids = append(ids, id)
+		}
+	}
+	if len(ids) > maxReferences {
+		ids = ids[len(ids)-maxReferences:]
+	}
+	return ids
+}

services/protocols/internal/application/usecase/message_headers_test.go

@@ -1,6 +1,10 @@
 package usecase
 
-import "testing"
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
 
 func TestExtractMessageHeaders_ReadsSubjectSenderAndSnippet(t *testing.T) {
 	msg := "From: Alice Example <alice@remote.test>\r\n" +
@@ -142,3 +146,37 @@ func TestExtractMessageHeaders_AttachmentDetection(t *testing.T) {
 		})
 	}
 }
+
+func TestHeadersCarryWhatAMessageAnswers(t *testing.T) {
+	raw := "Message-ID: <c@x.test>\r\nIn-Reply-To: <b@x.test>\r\nReferences: <a@x.test>\r\n <b@x.test>\r\nSubject: Re: Plan\r\n\r\nbody"
+	h := ExtractMessageHeaders([]byte(raw))
+	if h.InReplyTo != "b@x.test" {
+		t.Errorf("InReplyTo = %q, want b@x.test", h.InReplyTo)
+	}
+	if len(h.References) != 2 || h.References[0] != "a@x.test" || h.References[1] != "b@x.test" {
+		t.Errorf("References = %v, want [a@x.test b@x.test]", h.References)
+	}
+	if h.MessageID != "c@x.test" {
+		t.Errorf("MessageID = %q", h.MessageID)
+	}
+}
+
+func TestAMessageThatAnswersNothingHasNoReferences(t *testing.T) {
+	h := ExtractMessageHeaders([]byte("Subject: Hi\r\n\r\nbody"))
+	if h.InReplyTo != "" || len(h.References) != 0 {
+		t.Errorf("a first message claims to answer %q %v", h.InReplyTo, h.References)
+	}
+}
+
+// A chain can run to hundreds of ids. The ones nearest the end name the parent.
+func TestAVeryLongChainKeepsTheNewestIDs(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("References:")
+	for i := 0; i < 50; i++ {
+		fmt.Fprintf(&b, " <m%d@x.test>", i)
+	}
+	h := ExtractMessageHeaders([]byte(b.String() + "\r\nSubject: x\r\n\r\nbody"))
+	if len(h.References) != maxReferences || h.References[len(h.References)-1] != "m49@x.test" {
+		t.Errorf("kept %d ids ending %q", len(h.References), h.References[len(h.References)-1])
+	}
+}

services/protocols/internal/application/usecase/process_inbound_email.go

@@ -347,6 +347,8 @@ func (uc *ProcessInboundEmailUseCase) Handle(ctx context.Context, input ProcessI
 			FromDisplayName:  headers.FromDisplayName,
 			MessageIDHeader:  headers.MessageID,
 			HasAttachments:   headers.HasAttachments,
+			InReplyTo:        headers.InReplyTo,
+			References:       headers.References,
 			AlreadySeen:      alreadySeen,
 			SPFResult:        authResult.SPF,
 			DKIMResult:       authResult.DKIM,

services/protocols/internal/application/usecase/process_outbound_email.go

@@ -187,6 +187,8 @@ func (uc *ProcessOutboundEmailUseCase) fileSentCopy(ctx context.Context, input P
 		FromDisplayName:  headers.FromDisplayName,
 		MessageIDHeader:  headers.MessageID,
 		HasAttachments:   headers.HasAttachments,
+		InReplyTo:        headers.InReplyTo,
+		References:       headers.References,
 		// The sender's own copy is not unread mail
 		AlreadySeen: true,
 	}})

services/protocols/internal/application/usecase/webmail_mail.go

@@ -37,6 +37,9 @@ type WebmailUseCase struct {
 	// the submission use case, not here.)
 	blobStore port.BlobStorage
 	messages  port.InboundMessageRepository
+	// conversations serves the conversation views and records which message a
+	// reply answered. Optional; see SetConversations.
+	conversations port.ConversationReader
 }
 
 func NewWebmailUseCase(
@@ -91,6 +94,42 @@ func (uc *WebmailUseCase) ListMessages(
 	return uc.repo.ListMessages(ctx, id, folder, search, limit, offset)
 }
 
+// SetConversations enables the conversation views. Optional: without it the
+// thread routes answer "not configured" and the webmail falls back to the
+// plain message list.
+func (uc *WebmailUseCase) SetConversations(c port.ConversationReader) {
+	uc.conversations = c
+}
+
+// ErrConversationsDisabled is returned when no conversation reader was wired.
+var ErrConversationsDisabled = errors.New("conversations are not enabled")
+
+func (uc *WebmailUseCase) ListThreads(
+	ctx context.Context, address, folder, search string, limit, offset int,
+) ([]port.WebmailThread, error) {
+	if uc.conversations == nil {
+		return nil, ErrConversationsDisabled
+	}
+	id, err := uc.mailboxID(ctx, address)
+	if err != nil {
+		return nil, err
+	}
+	return uc.conversations.ListThreads(ctx, id, folder, search, limit, offset)
+}
+
+func (uc *WebmailUseCase) ThreadMessages(
+	ctx context.Context, address, folder, threadID string,
+) ([]port.WebmailThreadMessage, error) {
+	if uc.conversations == nil {
+		return nil, ErrConversationsDisabled
+	}
+	id, err := uc.mailboxID(ctx, address)
+	if err != nil {
+		return nil, err
+	}
+	return uc.conversations.ThreadMessages(ctx, id, folder, threadID)
+}
+
 // GetMessage returns the raw RFC 5322 bytes and marks the message read.
 func (uc *WebmailUseCase) GetMessage(ctx context.Context, address, folder string, uid uint32) ([]byte, error) {
 	id, err := uc.mailboxID(ctx, address)
@@ -207,6 +246,16 @@ func (uc *WebmailUseCase) Send(ctx context.Context, input ComposeInput) error {
 	// thing. Swallowed like the Sent copy above and for the same reason: the
 	// mail is already queued, and failing the request now would only invite a
 	// second copy to be sent.
+	// The message this one answers is marked answered. Swallowed for the same
+	// reason: the reply is already on its way, and a failed flag only means the
+	// list shows the conversation as unanswered until the next message.
+	if uc.conversations != nil && cleanMessageID(input.InReplyTo) != "" {
+		if mailboxID, idErr := uc.mailboxID(ctx, account.EmailAddress); idErr == nil {
+			if err := uc.conversations.MarkAnsweredByMessageID(ctx, mailboxID, input.InReplyTo); err != nil {
+				log.Printf("webmail: could not mark %s answered: %v", input.InReplyTo, err)
+			}
+		}
+	}
 	if input.DraftUID != 0 {
 		if mailboxID, idErr := uc.mailboxID(ctx, account.EmailAddress); idErr == nil {
 			if err := uc.repo.Expunge(ctx, mailboxID, "Drafts", input.DraftUID); err != nil {
@@ -341,6 +390,8 @@ func (uc *WebmailUseCase) SaveDraft(ctx context.Context, input ComposeInput, rep
 		FromDisplayName:  headers.FromDisplayName,
 		MessageIDHeader:  headers.MessageID,
 		HasAttachments:   headers.HasAttachments,
+		InReplyTo:        headers.InReplyTo,
+		References:       headers.References,
 		AlreadySeen:      true,
 	}})
 	if err != nil {