Lucas Rayan
e63b68acb2
Runs each state change through IMAP and through the webmail and checks the cached numbers against the rows. It fails today where the two stacks drifted: IMAP never moved the unread counter, and the webmail never gave the quota back.
406 lines
16 KiB
Go
406 lines
16 KiB
Go
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"lambdamail/protocols/internal/application/port"
|
|
)
|
|
|
|
// The mailbox state contract.
|
|
//
|
|
// A message can be read, flagged, moved, copied and deleted from IMAP, from the
|
|
// webmail, and by the server itself. All of them change the same rows, and the
|
|
// rows carry cached numbers (folder counters, the folder's modseq, the
|
|
// mailbox's used bytes) that every path must keep true. Before there was one
|
|
// implementation, each path kept them true on its own, and they drifted:
|
|
//
|
|
// - reading a message over IMAP never moved the unread counter, so the badge
|
|
// in the webmail and IMAP STATUS disagreed with the flags;
|
|
// - deleting from the webmail never gave the quota back;
|
|
// - a second expunge of the same message could push a counter below zero.
|
|
//
|
|
// These tests state what must hold after any operation and run every operation
|
|
// through each path that can perform it. They do not care which code did the
|
|
// work, only that the result is the same.
|
|
|
|
// folderState is everything observable about one folder.
|
|
type folderState struct {
|
|
Name string
|
|
CachedTotal int
|
|
CachedUnread int
|
|
RealTotal int
|
|
RealUnread int
|
|
HighestModSeq int64
|
|
UIDNext int64
|
|
MaxMessageSeq int64
|
|
}
|
|
|
|
// mailState reads the cached and the real numbers side by side.
|
|
func mailState(t *testing.T, pool *pgxpool.Pool, mailboxID string) (folders map[string]folderState, usedBytes, realBytes int64) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
|
|
rows, err := pool.Query(ctx, `
|
|
SELECT f.name, f.total_count, f.unread_count, f.highest_modseq, f.uid_next,
|
|
COUNT(e.id)::int,
|
|
COUNT(e.id) FILTER (WHERE NOT EXISTS (
|
|
SELECT 1 FROM message_flags mf
|
|
WHERE mf.message_id = e.id AND mf.received_at = e.received_at AND mf.flag = '\Seen'))::int,
|
|
COALESCE(MAX(e.modseq), 0)
|
|
FROM folders f
|
|
LEFT JOIN email_messages e ON e.folder_id = f.id AND e.expunged_at IS NULL
|
|
WHERE f.mailbox_id = $1
|
|
GROUP BY f.id`, mailboxID)
|
|
if err != nil {
|
|
t.Fatalf("read folder state: %v", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
folders = map[string]folderState{}
|
|
for rows.Next() {
|
|
var s folderState
|
|
if err := rows.Scan(&s.Name, &s.CachedTotal, &s.CachedUnread, &s.HighestModSeq, &s.UIDNext, &s.RealTotal, &s.RealUnread, &s.MaxMessageSeq); err != nil {
|
|
t.Fatalf("scan folder state: %v", err)
|
|
}
|
|
folders[s.Name] = s
|
|
}
|
|
|
|
if err := pool.QueryRow(ctx, `SELECT used_bytes FROM mailboxes WHERE id = $1`, mailboxID).Scan(&usedBytes); err != nil {
|
|
t.Fatalf("read used_bytes: %v", err)
|
|
}
|
|
if err := pool.QueryRow(ctx, `SELECT COALESCE(SUM(size_bytes), 0) FROM email_messages WHERE mailbox_id = $1 AND expunged_at IS NULL`, mailboxID).Scan(&realBytes); err != nil {
|
|
t.Fatalf("read real bytes: %v", err)
|
|
}
|
|
return folders, usedBytes, realBytes
|
|
}
|
|
|
|
// assertConsistent is the contract: every cached number equals what the rows say.
|
|
func assertConsistent(t *testing.T, pool *pgxpool.Pool, mailboxID, after string) {
|
|
t.Helper()
|
|
folders, used, real := mailState(t, pool, mailboxID)
|
|
|
|
for name, f := range folders {
|
|
if f.CachedTotal != f.RealTotal {
|
|
t.Errorf("after %s: %s total_count = %d, but it holds %d messages", after, name, f.CachedTotal, f.RealTotal)
|
|
}
|
|
if f.CachedUnread != f.RealUnread {
|
|
t.Errorf("after %s: %s unread_count = %d, but %d messages are unread", after, name, f.CachedUnread, f.RealUnread)
|
|
}
|
|
if f.CachedUnread < 0 || f.CachedTotal < 0 {
|
|
t.Errorf("after %s: %s has a negative counter (unread %d, total %d)", after, name, f.CachedUnread, f.CachedTotal)
|
|
}
|
|
if f.HighestModSeq < f.MaxMessageSeq {
|
|
t.Errorf("after %s: %s highest_modseq %d is behind a message's modseq %d, so a CONDSTORE client would miss the change", after, name, f.HighestModSeq, f.MaxMessageSeq)
|
|
}
|
|
}
|
|
if used != real {
|
|
t.Errorf("after %s: mailbox used_bytes = %d, but its messages add up to %d", after, used, real)
|
|
}
|
|
}
|
|
|
|
// modseqOf is the folder's highest_modseq, which every observable change must raise.
|
|
func modseqOf(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string) int64 {
|
|
t.Helper()
|
|
folders, _, _ := mailState(t, pool, mailboxID)
|
|
return folders[folder].HighestModSeq
|
|
}
|
|
|
|
func folderID(t *testing.T, pool *pgxpool.Pool, mailboxID, name string) string {
|
|
t.Helper()
|
|
var id string
|
|
if err := pool.QueryRow(context.Background(), `SELECT id::text FROM folders WHERE mailbox_id = $1 AND name = $2`, mailboxID, name).Scan(&id); err != nil {
|
|
t.Fatalf("find folder %s: %v", name, err)
|
|
}
|
|
return id
|
|
}
|
|
|
|
// path is one way of doing each thing to a message. IMAP and webmail both
|
|
// implement it; the contract runs against each.
|
|
type path struct {
|
|
name string
|
|
setSeen func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32, seen bool)
|
|
setFlag func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32, flagged bool)
|
|
expunge func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32)
|
|
moveTo func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32, target string)
|
|
flagsKept func() bool
|
|
}
|
|
|
|
func imapPath() path {
|
|
return path{
|
|
name: "imap",
|
|
setSeen: func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32, seen bool) {
|
|
op := port.FlagOpAdd
|
|
if !seen {
|
|
op = port.FlagOpDel
|
|
}
|
|
if _, err := NewFlagRepository(pool).SetFlags(context.Background(), folderID(t, pool, mailboxID, folder), uid, op, []string{`\Seen`}, 0); err != nil {
|
|
t.Fatalf("imap SetFlags: %v", err)
|
|
}
|
|
},
|
|
setFlag: func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32, flagged bool) {
|
|
op := port.FlagOpAdd
|
|
if !flagged {
|
|
op = port.FlagOpDel
|
|
}
|
|
if _, err := NewFlagRepository(pool).SetFlags(context.Background(), folderID(t, pool, mailboxID, folder), uid, op, []string{`\Flagged`}, 0); err != nil {
|
|
t.Fatalf("imap SetFlags: %v", err)
|
|
}
|
|
},
|
|
expunge: func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32) {
|
|
if err := NewExpungeRepository(pool).Expunge(context.Background(), folderID(t, pool, mailboxID, folder), []uint32{uid}); err != nil {
|
|
t.Fatalf("imap Expunge: %v", err)
|
|
}
|
|
},
|
|
moveTo: func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32, target string) {
|
|
// IMAP has no MOVE here: a client copies, marks \Deleted and expunges
|
|
src, dst := folderID(t, pool, mailboxID, folder), folderID(t, pool, mailboxID, target)
|
|
if _, err := NewCopyRepository(pool).CopyMessages(context.Background(), src, []uint32{uid}, dst); err != nil {
|
|
t.Fatalf("imap Copy: %v", err)
|
|
}
|
|
if err := NewExpungeRepository(pool).Expunge(context.Background(), src, []uint32{uid}); err != nil {
|
|
t.Fatalf("imap Expunge after copy: %v", err)
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
func webmailPath() path {
|
|
return path{
|
|
name: "webmail",
|
|
setSeen: func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32, seen bool) {
|
|
if err := NewWebmailRepository(pool).MarkSeen(context.Background(), mailboxID, folder, uid, seen); err != nil {
|
|
t.Fatalf("webmail MarkSeen: %v", err)
|
|
}
|
|
},
|
|
setFlag: func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32, flagged bool) {
|
|
if err := NewWebmailRepository(pool).MarkFlagged(context.Background(), mailboxID, folder, uid, flagged); err != nil {
|
|
t.Fatalf("webmail MarkFlagged: %v", err)
|
|
}
|
|
},
|
|
expunge: func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32) {
|
|
if err := NewWebmailRepository(pool).Expunge(context.Background(), mailboxID, folder, uid); err != nil {
|
|
t.Fatalf("webmail Expunge: %v", err)
|
|
}
|
|
},
|
|
moveTo: func(t *testing.T, pool *pgxpool.Pool, mailboxID, folder string, uid uint32, target string) {
|
|
if _, err := NewWebmailRepository(pool).MoveToFolder(context.Background(), mailboxID, folder, uid, target); err != nil {
|
|
t.Fatalf("webmail Move: %v", err)
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
func eachPath(t *testing.T, scenario func(t *testing.T, p path)) {
|
|
for _, p := range []path{imapPath(), webmailPath()} {
|
|
p := p
|
|
t.Run(p.name, func(t *testing.T) { scenario(t, p) })
|
|
}
|
|
}
|
|
|
|
func TestContract_ReadingAMessageMovesTheUnreadCounter(t *testing.T) {
|
|
pool := testPool(t)
|
|
eachPath(t, func(t *testing.T, p path) {
|
|
mailboxID := seedMailbox(t, pool)
|
|
a := deliver(t, pool, mailboxID, "INBOX", "a", false)
|
|
deliver(t, pool, mailboxID, "INBOX", "b", false)
|
|
assertConsistent(t, pool, mailboxID, "delivery")
|
|
|
|
before := modseqOf(t, pool, mailboxID, "INBOX")
|
|
p.setSeen(t, pool, mailboxID, "INBOX", a, true)
|
|
assertConsistent(t, pool, mailboxID, "marking one message read")
|
|
if after := modseqOf(t, pool, mailboxID, "INBOX"); after <= before {
|
|
t.Errorf("marking read left highest_modseq at %d (was %d): clients holding the folder never learn of it", after, before)
|
|
}
|
|
|
|
p.setSeen(t, pool, mailboxID, "INBOX", a, true)
|
|
assertConsistent(t, pool, mailboxID, "marking the same message read twice")
|
|
|
|
p.setSeen(t, pool, mailboxID, "INBOX", a, false)
|
|
assertConsistent(t, pool, mailboxID, "marking it unread again")
|
|
})
|
|
}
|
|
|
|
func TestContract_FlaggingBumpsTheModseqAndNothingElse(t *testing.T) {
|
|
pool := testPool(t)
|
|
eachPath(t, func(t *testing.T, p path) {
|
|
mailboxID := seedMailbox(t, pool)
|
|
a := deliver(t, pool, mailboxID, "INBOX", "a", false)
|
|
|
|
before := modseqOf(t, pool, mailboxID, "INBOX")
|
|
p.setFlag(t, pool, mailboxID, "INBOX", a, true)
|
|
assertConsistent(t, pool, mailboxID, "flagging")
|
|
if after := modseqOf(t, pool, mailboxID, "INBOX"); after <= before {
|
|
t.Errorf("flagging left highest_modseq at %d (was %d)", after, before)
|
|
}
|
|
p.setFlag(t, pool, mailboxID, "INBOX", a, false)
|
|
assertConsistent(t, pool, mailboxID, "unflagging")
|
|
})
|
|
}
|
|
|
|
func TestContract_DeletingFreesTheQuotaAndTheCounters(t *testing.T) {
|
|
pool := testPool(t)
|
|
eachPath(t, func(t *testing.T, p path) {
|
|
mailboxID := seedMailbox(t, pool)
|
|
unread := deliver(t, pool, mailboxID, "INBOX", "unread", false)
|
|
read := deliver(t, pool, mailboxID, "INBOX", "read", true)
|
|
assertConsistent(t, pool, mailboxID, "delivery")
|
|
|
|
p.expunge(t, pool, mailboxID, "INBOX", unread)
|
|
assertConsistent(t, pool, mailboxID, "deleting an unread message")
|
|
|
|
p.expunge(t, pool, mailboxID, "INBOX", read)
|
|
assertConsistent(t, pool, mailboxID, "deleting a read message")
|
|
|
|
_, used, _ := mailState(t, pool, mailboxID)
|
|
if used != 0 {
|
|
t.Errorf("an emptied mailbox still reports %d used bytes", used)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestContract_AMessageAlreadyDeletedCannotDriveACounterNegative(t *testing.T) {
|
|
pool := testPool(t)
|
|
eachPath(t, func(t *testing.T, p path) {
|
|
mailboxID := seedMailbox(t, pool)
|
|
a := deliver(t, pool, mailboxID, "INBOX", "a", false)
|
|
deliver(t, pool, mailboxID, "INBOX", "b", false)
|
|
|
|
p.expunge(t, pool, mailboxID, "INBOX", a)
|
|
// The second one is the same message again. The webmail reports it as
|
|
// not found; IMAP ignores it. Either way no number may move.
|
|
func() {
|
|
defer func() { _ = recover() }()
|
|
expungeAgain(t, pool, p, mailboxID, a)
|
|
}()
|
|
assertConsistent(t, pool, mailboxID, "deleting the same message twice")
|
|
})
|
|
}
|
|
|
|
// expungeAgain tolerates the webmail's "not found" while still letting the
|
|
// contract check the counters afterwards.
|
|
func expungeAgain(t *testing.T, pool *pgxpool.Pool, p path, mailboxID string, uid uint32) {
|
|
t.Helper()
|
|
switch p.name {
|
|
case "imap":
|
|
_ = NewExpungeRepository(pool).Expunge(context.Background(), folderID(t, pool, mailboxID, "INBOX"), []uint32{uid})
|
|
default:
|
|
_ = NewWebmailRepository(pool).Expunge(context.Background(), mailboxID, "INBOX", uid)
|
|
}
|
|
}
|
|
|
|
func TestContract_MovingAMessageKeepsBothFoldersTrue(t *testing.T) {
|
|
pool := testPool(t)
|
|
eachPath(t, func(t *testing.T, p path) {
|
|
mailboxID := seedMailbox(t, pool)
|
|
unread := deliver(t, pool, mailboxID, "INBOX", "unread", false)
|
|
read := deliver(t, pool, mailboxID, "INBOX", "read", true)
|
|
|
|
p.moveTo(t, pool, mailboxID, "INBOX", unread, "Trash")
|
|
assertConsistent(t, pool, mailboxID, "moving an unread message")
|
|
p.moveTo(t, pool, mailboxID, "INBOX", read, "Trash")
|
|
assertConsistent(t, pool, mailboxID, "moving a read message")
|
|
|
|
folders, _, _ := mailState(t, pool, mailboxID)
|
|
if folders["INBOX"].RealTotal != 0 || folders["Trash"].RealTotal != 2 {
|
|
t.Errorf("INBOX holds %d and Trash %d, want 0 and 2", folders["INBOX"].RealTotal, folders["Trash"].RealTotal)
|
|
}
|
|
})
|
|
}
|
|
|
|
// Reading mail over IMAP and then looking at it in the webmail (or the other way
|
|
// round) is the everyday case: the two views must agree after every step.
|
|
func TestContract_ImapAndWebmailAgreeOnTheSameMailbox(t *testing.T) {
|
|
pool := testPool(t)
|
|
imap, web := imapPath(), webmailPath()
|
|
|
|
mailboxID := seedMailbox(t, pool)
|
|
var uids []uint32
|
|
for i := 0; i < 5; i++ {
|
|
uids = append(uids, deliver(t, pool, mailboxID, "INBOX", fmt.Sprintf("m%d", i), false))
|
|
}
|
|
|
|
imap.setSeen(t, pool, mailboxID, "INBOX", uids[0], true)
|
|
assertConsistent(t, pool, mailboxID, "reading m0 over IMAP")
|
|
web.setSeen(t, pool, mailboxID, "INBOX", uids[1], true)
|
|
assertConsistent(t, pool, mailboxID, "reading m1 in the webmail")
|
|
imap.setFlag(t, pool, mailboxID, "INBOX", uids[2], true)
|
|
web.expunge(t, pool, mailboxID, "INBOX", uids[3])
|
|
assertConsistent(t, pool, mailboxID, "deleting m3 in the webmail")
|
|
imap.expunge(t, pool, mailboxID, "INBOX", uids[1])
|
|
assertConsistent(t, pool, mailboxID, "deleting the already-read m1 over IMAP")
|
|
web.moveTo(t, pool, mailboxID, "INBOX", uids[4], "Trash")
|
|
assertConsistent(t, pool, mailboxID, "moving m4 to Trash in the webmail")
|
|
imap.moveTo(t, pool, mailboxID, "INBOX", uids[2], "Trash")
|
|
assertConsistent(t, pool, mailboxID, "moving m2 to Trash over IMAP")
|
|
|
|
// What the webmail reports for a folder is what IMAP would answer
|
|
ctx := context.Background()
|
|
wmFolders, err := NewWebmailRepository(pool).ListFolders(ctx, mailboxID)
|
|
if err != nil {
|
|
t.Fatalf("webmail ListFolders: %v", err)
|
|
}
|
|
imFolders, err := NewImapFolderRepository(pool).ListFolders(ctx, mailboxID)
|
|
if err != nil {
|
|
t.Fatalf("imap ListFolders: %v", err)
|
|
}
|
|
imapTotals := map[string]int{}
|
|
for _, f := range imFolders {
|
|
imapTotals[f.Name] = int(f.NumMessages)
|
|
}
|
|
for _, f := range wmFolders {
|
|
if got := imapTotals[f.Name]; got != f.TotalCount {
|
|
t.Errorf("%s: the webmail shows %d messages and IMAP says %d", f.Name, f.TotalCount, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A folder deleted by its owner takes its messages with it, and their bytes.
|
|
func TestContract_DeletingAFolderReturnsItsBytes(t *testing.T) {
|
|
pool := testPool(t)
|
|
ctx := context.Background()
|
|
mailboxID := seedMailbox(t, pool)
|
|
repo := NewWebmailRepository(pool)
|
|
|
|
if err := repo.CreateFolder(ctx, mailboxID, "Projetos"); err != nil {
|
|
t.Fatalf("create folder: %v", err)
|
|
}
|
|
deliver(t, pool, mailboxID, "Projetos", "one", false)
|
|
deliver(t, pool, mailboxID, "Projetos", "two", true)
|
|
deliver(t, pool, mailboxID, "INBOX", "stays", false)
|
|
assertConsistent(t, pool, mailboxID, "filing into a new folder")
|
|
|
|
if err := repo.DeleteFolder(ctx, mailboxID, "Projetos"); err != nil {
|
|
t.Fatalf("delete folder: %v", err)
|
|
}
|
|
assertConsistent(t, pool, mailboxID, "deleting the folder")
|
|
}
|
|
|
|
// IMAP's STORE can replace a message's whole flag set; the unread counter has to
|
|
// follow the \Seen that comes or goes with it.
|
|
func TestContract_ReplacingTheFlagSetKeepsTheUnreadCounterTrue(t *testing.T) {
|
|
pool := testPool(t)
|
|
ctx := context.Background()
|
|
mailboxID := seedMailbox(t, pool)
|
|
a := deliver(t, pool, mailboxID, "INBOX", "a", false)
|
|
b := deliver(t, pool, mailboxID, "INBOX", "b", true)
|
|
inbox := folderID(t, pool, mailboxID, "INBOX")
|
|
repo := NewFlagRepository(pool)
|
|
|
|
if _, err := repo.SetFlags(ctx, inbox, a, port.FlagOpSet, []string{`\Seen`, `\Flagged`}, 0); err != nil {
|
|
t.Fatalf("SetFlags: %v", err)
|
|
}
|
|
assertConsistent(t, pool, mailboxID, "replacing flags with one that includes \\Seen")
|
|
|
|
if _, err := repo.SetFlags(ctx, inbox, b, port.FlagOpSet, []string{`\Flagged`}, 0); err != nil {
|
|
t.Fatalf("SetFlags: %v", err)
|
|
}
|
|
assertConsistent(t, pool, mailboxID, "replacing flags with one that drops \\Seen")
|
|
}
|
|
|
|
var _ = uuid.Nil
|