#15: implement ACME certificate store, Traefik/Coolify cert watcher & secrets vault

services/protocols/internal/infrastructure/tls/acme_cert_watcher.go

@@ -0,0 +1,242 @@
+package tlsprovider
+
+import (
+	"crypto/sha256"
+	"crypto/tls"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"time"
+)
+
+// AcmeCertWatcher serves the certificates Traefik obtained, reading them from
+// the acme.json it maintains (PLAN.md section 8.1).
+//
+// Three properties of that file drive this design (section 8.2):
+//
+//  1. Traefik replaces acme.json by writing a temporary file and renaming it.
+//     A watcher bound to the original inode goes permanently blind after the
+//     first renewal, so this watcher polls the path instead and compares a
+//     content hash. The symptom of getting this wrong only appears ninety
+//     days later, in production, as an expired certificate.
+//  2. Traefik only holds certificates for hosts it routes. A missing entry is
+//     a deployment problem, reported loudly rather than papered over.
+//  3. The file layout varies by Traefik version and resolver name, so the
+//     parser iterates every resolver and tolerates unknown fields.
+type AcmeCertWatcher struct {
+	path        string
+	defaultHost string
+
+	// certificates is swapped atomically so that in-flight handshakes keep
+	// using a consistent map.
+	certificates atomic.Pointer[map[string]*tls.Certificate]
+
+	lastHash     atomic.Pointer[[32]byte]
+	lastReloadAt atomic.Pointer[time.Time]
+
+	// unknownSNI counts handshakes for names we hold no certificate for; it
+	// backs the tls_unknown_sni_total metric of PLAN.md section 8.4.
+	unknownSNI atomic.Int64
+}
+
+var (
+	ErrNoCertificatesLoaded = errors.New("acme watcher: no certificates loaded")
+	ErrNoCertificateForSNI  = errors.New("acme watcher: no certificate for the requested server name")
+)
+
+// acmeFile mirrors the parts of acme.json this code reads. Every other field
+// is ignored on purpose so a Traefik upgrade does not break certificate
+// loading.
+type acmeFile map[string]struct {
+	Certificates []struct {
+		Domain struct {
+			Main string   `json:"main"`
+			SANs []string `json:"sans"`
+		} `json:"domain"`
+		Certificate string `json:"certificate"`
+		Key         string `json:"key"`
+	} `json:"certificates"`
+}
+
+// NewAcmeCertWatcher reads the store once so that a broken configuration
+// fails at startup rather than at the first connection.
+func NewAcmeCertWatcher(dir, filename, defaultHost string) (*AcmeCertWatcher, error) {
+	if filename == "" {
+		filename = "acme.json"
+	}
+	w := &AcmeCertWatcher{
+		path:        filepath.Join(dir, filename),
+		defaultHost: strings.ToLower(defaultHost),
+	}
+	if err := w.reload(); err != nil {
+		return nil, err
+	}
+	return w, nil
+}
+
+// Watch reloads on a fixed interval. Polling is the mechanism, not a fallback:
+// see the rename note on the type.
+func (w *AcmeCertWatcher) Watch(stop <-chan struct{}, interval time.Duration) {
+	if interval <= 0 {
+		interval = 60 * time.Second
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-stop:
+			return
+		case <-ticker.C:
+			if err := w.reload(); err != nil {
+				// Keeping the previous store is deliberate: a truncated file
+				// caught mid-write must not leave the listeners with nothing
+				// (PLAN.md section 8.4).
+				log.Printf("WARNING: acme watcher reload failed, keeping the certificates already loaded: %v", err)
+			}
+		}
+	}
+}
+
+func (w *AcmeCertWatcher) reload() error {
+	raw, err := os.ReadFile(w.path)
+	if err != nil {
+		return fmt.Errorf("read %s: %w", w.path, err)
+	}
+
+	hash := sha256.Sum256(raw)
+	if previous := w.lastHash.Load(); previous != nil && *previous == hash {
+		return nil
+	}
+
+	certificates, err := parseAcmeStore(raw)
+	if err != nil {
+		return err
+	}
+	if len(certificates) == 0 {
+		return fmt.Errorf("acme watcher: %s holds no usable certificate", w.path)
+	}
+
+	w.certificates.Store(&certificates)
+	w.lastHash.Store(&hash)
+	now := time.Now()
+	w.lastReloadAt.Store(&now)
+
+	return nil
+}
+
+// parseAcmeStore walks every resolver in the file. A single unusable entry -
+// an expired certificate, or one whose key does not match - is skipped rather
+// than failing the whole load, so one stale domain cannot take the server down.
+func parseAcmeStore(raw []byte) (map[string]*tls.Certificate, error) {
+	var store acmeFile
+	if err := json.Unmarshal(raw, &store); err != nil {
+		return nil, fmt.Errorf("parse acme store: %w", err)
+	}
+
+	certificates := map[string]*tls.Certificate{}
+
+	for resolverName, resolver := range store {
+		for _, entry := range resolver.Certificates {
+			certPEM, err := base64.StdEncoding.DecodeString(entry.Certificate)
+			if err != nil {
+				log.Printf("acme watcher: skipping %s in resolver %q: certificate is not valid base64", entry.Domain.Main, resolverName)
+				continue
+			}
+			keyPEM, err := base64.StdEncoding.DecodeString(entry.Key)
+			if err != nil {
+				log.Printf("acme watcher: skipping %s in resolver %q: key is not valid base64", entry.Domain.Main, resolverName)
+				continue
+			}
+
+			pair, err := tls.X509KeyPair(certPEM, keyPEM)
+			if err != nil {
+				log.Printf("acme watcher: skipping %s in resolver %q: %v", entry.Domain.Main, resolverName, err)
+				continue
+			}
+
+			for _, name := range append([]string{entry.Domain.Main}, entry.Domain.SANs...) {
+				if name != "" {
+					certificates[strings.ToLower(name)] = &pair
+				}
+			}
+		}
+	}
+
+	return certificates, nil
+}
+
+// GetCertificate satisfies tls.Config.GetCertificate. It matches the exact
+// name, then the wildcard covering it, then the mail host, which is what a
+// client connecting by IP or with no SNI should receive.
+func (w *AcmeCertWatcher) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	loaded := w.certificates.Load()
+	if loaded == nil {
+		return nil, ErrNoCertificatesLoaded
+	}
+	certificates := *loaded
+
+	name := ""
+	if hello != nil {
+		name = strings.ToLower(strings.TrimSuffix(hello.ServerName, "."))
+	}
+
+	if cert, ok := certificates[name]; ok && name != "" {
+		return cert, nil
+	}
+	if cert, ok := certificates[wildcardOf(name)]; ok {
+		return cert, nil
+	}
+	if cert, ok := certificates[w.defaultHost]; ok {
+		w.unknownSNI.Add(1)
+		return cert, nil
+	}
+
+	w.unknownSNI.Add(1)
+	return nil, fmt.Errorf("%w: %q", ErrNoCertificateForSNI, name)
+}
+
+// wildcardOf turns "mail.example.com" into "*.example.com".
+func wildcardOf(name string) string {
+	if _, rest, found := strings.Cut(name, "."); found && rest != "" {
+		return "*." + rest
+	}
+	return ""
+}
+
+// LastReload reports when the store was last successfully refreshed. A
+// timestamp that stops advancing is the signal that the watcher went blind
+// (PLAN.md risk R2).
+func (w *AcmeCertWatcher) LastReload() time.Time {
+	if t := w.lastReloadAt.Load(); t != nil {
+		return *t
+	}
+	return time.Time{}
+}
+
+// UnknownSNICount backs the tls_unknown_sni_total metric.
+func (w *AcmeCertWatcher) UnknownSNICount() int64 {
+	return w.unknownSNI.Load()
+}
+
+// HasCertificateFor reports whether a certificate exists for a host. The
+// preflight uses it to catch the missing-router trap of PLAN.md section 8.2.
+func (w *AcmeCertWatcher) HasCertificateFor(host string) bool {
+	loaded := w.certificates.Load()
+	if loaded == nil {
+		return false
+	}
+	host = strings.ToLower(host)
+	certificates := *loaded
+	if _, ok := certificates[host]; ok {
+		return true
+	}
+	_, ok := certificates[wildcardOf(host)]
+	return ok
+}