#15: implement ACME certificate store, Traefik/Coolify cert watcher & secrets vault
This commit is contained in:
Lucas Rayan committed
1 parent
769e0b59f0
commit
6ef427dc09
11 files changed
+1667
No files matched your search
@@ -0,0 +1,242 @@
|
||||
package tlsprovider
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AcmeCertWatcher serves the certificates Traefik obtained, reading them from
|
||||
// the acme.json it maintains (PLAN.md section 8.1).
|
||||
//
|
||||
// Three properties of that file drive this design (section 8.2):
|
||||
//
|
||||
// 1. Traefik replaces acme.json by writing a temporary file and renaming it.
|
||||
// A watcher bound to the original inode goes permanently blind after the
|
||||
// first renewal, so this watcher polls the path instead and compares a
|
||||
// content hash. The symptom of getting this wrong only appears ninety
|
||||
// days later, in production, as an expired certificate.
|
||||
// 2. Traefik only holds certificates for hosts it routes. A missing entry is
|
||||
// a deployment problem, reported loudly rather than papered over.
|
||||
// 3. The file layout varies by Traefik version and resolver name, so the
|
||||
// parser iterates every resolver and tolerates unknown fields.
|
||||
type AcmeCertWatcher struct {
|
||||
path string
|
||||
defaultHost string
|
||||
|
||||
// certificates is swapped atomically so that in-flight handshakes keep
|
||||
// using a consistent map.
|
||||
certificates atomic.Pointer[map[string]*tls.Certificate]
|
||||
|
||||
lastHash atomic.Pointer[[32]byte]
|
||||
lastReloadAt atomic.Pointer[time.Time]
|
||||
|
||||
// unknownSNI counts handshakes for names we hold no certificate for; it
|
||||
// backs the tls_unknown_sni_total metric of PLAN.md section 8.4.
|
||||
unknownSNI atomic.Int64
|
||||
}
|
||||
|
||||
var (
|
||||
ErrNoCertificatesLoaded = errors.New("acme watcher: no certificates loaded")
|
||||
ErrNoCertificateForSNI = errors.New("acme watcher: no certificate for the requested server name")
|
||||
)
|
||||
|
||||
// acmeFile mirrors the parts of acme.json this code reads. Every other field
|
||||
// is ignored on purpose so a Traefik upgrade does not break certificate
|
||||
// loading.
|
||||
type acmeFile map[string]struct {
|
||||
Certificates []struct {
|
||||
Domain struct {
|
||||
Main string `json:"main"`
|
||||
SANs []string `json:"sans"`
|
||||
} `json:"domain"`
|
||||
Certificate string `json:"certificate"`
|
||||
Key string `json:"key"`
|
||||
} `json:"certificates"`
|
||||
}
|
||||
|
||||
// NewAcmeCertWatcher reads the store once so that a broken configuration
|
||||
// fails at startup rather than at the first connection.
|
||||
func NewAcmeCertWatcher(dir, filename, defaultHost string) (*AcmeCertWatcher, error) {
|
||||
if filename == "" {
|
||||
filename = "acme.json"
|
||||
}
|
||||
w := &AcmeCertWatcher{
|
||||
path: filepath.Join(dir, filename),
|
||||
defaultHost: strings.ToLower(defaultHost),
|
||||
}
|
||||
if err := w.reload(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Watch reloads on a fixed interval. Polling is the mechanism, not a fallback:
|
||||
// see the rename note on the type.
|
||||
func (w *AcmeCertWatcher) Watch(stop <-chan struct{}, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = 60 * time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := w.reload(); err != nil {
|
||||
// Keeping the previous store is deliberate: a truncated file
|
||||
// caught mid-write must not leave the listeners with nothing
|
||||
// (PLAN.md section 8.4).
|
||||
log.Printf("WARNING: acme watcher reload failed, keeping the certificates already loaded: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *AcmeCertWatcher) reload() error {
|
||||
raw, err := os.ReadFile(w.path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read %s: %w", w.path, err)
|
||||
}
|
||||
|
||||
hash := sha256.Sum256(raw)
|
||||
if previous := w.lastHash.Load(); previous != nil && *previous == hash {
|
||||
return nil
|
||||
}
|
||||
|
||||
certificates, err := parseAcmeStore(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(certificates) == 0 {
|
||||
return fmt.Errorf("acme watcher: %s holds no usable certificate", w.path)
|
||||
}
|
||||
|
||||
w.certificates.Store(&certificates)
|
||||
w.lastHash.Store(&hash)
|
||||
now := time.Now()
|
||||
w.lastReloadAt.Store(&now)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseAcmeStore walks every resolver in the file. A single unusable entry -
|
||||
// an expired certificate, or one whose key does not match - is skipped rather
|
||||
// than failing the whole load, so one stale domain cannot take the server down.
|
||||
func parseAcmeStore(raw []byte) (map[string]*tls.Certificate, error) {
|
||||
var store acmeFile
|
||||
if err := json.Unmarshal(raw, &store); err != nil {
|
||||
return nil, fmt.Errorf("parse acme store: %w", err)
|
||||
}
|
||||
|
||||
certificates := map[string]*tls.Certificate{}
|
||||
|
||||
for resolverName, resolver := range store {
|
||||
for _, entry := range resolver.Certificates {
|
||||
certPEM, err := base64.StdEncoding.DecodeString(entry.Certificate)
|
||||
if err != nil {
|
||||
log.Printf("acme watcher: skipping %s in resolver %q: certificate is not valid base64", entry.Domain.Main, resolverName)
|
||||
continue
|
||||
}
|
||||
keyPEM, err := base64.StdEncoding.DecodeString(entry.Key)
|
||||
if err != nil {
|
||||
log.Printf("acme watcher: skipping %s in resolver %q: key is not valid base64", entry.Domain.Main, resolverName)
|
||||
continue
|
||||
}
|
||||
|
||||
pair, err := tls.X509KeyPair(certPEM, keyPEM)
|
||||
if err != nil {
|
||||
log.Printf("acme watcher: skipping %s in resolver %q: %v", entry.Domain.Main, resolverName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, name := range append([]string{entry.Domain.Main}, entry.Domain.SANs...) {
|
||||
if name != "" {
|
||||
certificates[strings.ToLower(name)] = &pair
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return certificates, nil
|
||||
}
|
||||
|
||||
// GetCertificate satisfies tls.Config.GetCertificate. It matches the exact
|
||||
// name, then the wildcard covering it, then the mail host, which is what a
|
||||
// client connecting by IP or with no SNI should receive.
|
||||
func (w *AcmeCertWatcher) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
loaded := w.certificates.Load()
|
||||
if loaded == nil {
|
||||
return nil, ErrNoCertificatesLoaded
|
||||
}
|
||||
certificates := *loaded
|
||||
|
||||
name := ""
|
||||
if hello != nil {
|
||||
name = strings.ToLower(strings.TrimSuffix(hello.ServerName, "."))
|
||||
}
|
||||
|
||||
if cert, ok := certificates[name]; ok && name != "" {
|
||||
return cert, nil
|
||||
}
|
||||
if cert, ok := certificates[wildcardOf(name)]; ok {
|
||||
return cert, nil
|
||||
}
|
||||
if cert, ok := certificates[w.defaultHost]; ok {
|
||||
w.unknownSNI.Add(1)
|
||||
return cert, nil
|
||||
}
|
||||
|
||||
w.unknownSNI.Add(1)
|
||||
return nil, fmt.Errorf("%w: %q", ErrNoCertificateForSNI, name)
|
||||
}
|
||||
|
||||
// wildcardOf turns "mail.example.com" into "*.example.com".
|
||||
func wildcardOf(name string) string {
|
||||
if _, rest, found := strings.Cut(name, "."); found && rest != "" {
|
||||
return "*." + rest
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// LastReload reports when the store was last successfully refreshed. A
|
||||
// timestamp that stops advancing is the signal that the watcher went blind
|
||||
// (PLAN.md risk R2).
|
||||
func (w *AcmeCertWatcher) LastReload() time.Time {
|
||||
if t := w.lastReloadAt.Load(); t != nil {
|
||||
return *t
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// UnknownSNICount backs the tls_unknown_sni_total metric.
|
||||
func (w *AcmeCertWatcher) UnknownSNICount() int64 {
|
||||
return w.unknownSNI.Load()
|
||||
}
|
||||
|
||||
// HasCertificateFor reports whether a certificate exists for a host. The
|
||||
// preflight uses it to catch the missing-router trap of PLAN.md section 8.2.
|
||||
func (w *AcmeCertWatcher) HasCertificateFor(host string) bool {
|
||||
loaded := w.certificates.Load()
|
||||
if loaded == nil {
|
||||
return false
|
||||
}
|
||||
host = strings.ToLower(host)
|
||||
certificates := *loaded
|
||||
if _, ok := certificates[host]; ok {
|
||||
return true
|
||||
}
|
||||
_, ok := certificates[wildcardOf(host)]
|
||||
return ok
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
package tlsprovider
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// makeCertPEM issues a self-signed certificate for the given names.
|
||||
func makeCertPEM(t *testing.T, names ...string) (certPEM, keyPEM []byte) {
|
||||
t.Helper()
|
||||
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("generate key: %v", err)
|
||||
}
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(time.Now().UnixNano()),
|
||||
Subject: pkix.Name{CommonName: names[0]},
|
||||
DNSNames: names,
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(90 * 24 * time.Hour),
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatalf("create certificate: %v", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(key)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal key: %v", err)
|
||||
}
|
||||
|
||||
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
|
||||
pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
|
||||
}
|
||||
|
||||
// writeAcmeStore builds an acme.json in the shape Traefik writes: a map of
|
||||
// resolver name to that resolver's account and certificates.
|
||||
func writeAcmeStore(t *testing.T, dir, resolverName string, entries map[string][]string) string {
|
||||
t.Helper()
|
||||
|
||||
type domain struct {
|
||||
Main string `json:"main"`
|
||||
SANs []string `json:"sans"`
|
||||
}
|
||||
type certificate struct {
|
||||
Domain domain `json:"domain"`
|
||||
Certificate string `json:"certificate"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
var certificates []certificate
|
||||
for main, sans := range entries {
|
||||
certPEM, keyPEM := makeCertPEM(t, append([]string{main}, sans...)...)
|
||||
certificates = append(certificates, certificate{
|
||||
Domain: domain{Main: main, SANs: sans},
|
||||
Certificate: base64.StdEncoding.EncodeToString(certPEM),
|
||||
Key: base64.StdEncoding.EncodeToString(keyPEM),
|
||||
})
|
||||
}
|
||||
|
||||
store := map[string]any{
|
||||
resolverName: map[string]any{
|
||||
// An Account block is always present and must be ignored.
|
||||
"Account": map[string]any{"Email": "ops@example.test"},
|
||||
"certificates": certificates,
|
||||
},
|
||||
}
|
||||
|
||||
raw, err := json.Marshal(store)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal store: %v", err)
|
||||
}
|
||||
|
||||
path := filepath.Join(dir, "acme.json")
|
||||
if err := os.WriteFile(path, raw, 0o600); err != nil {
|
||||
t.Fatalf("write store: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestAcmeCertWatcher_ServesCertificateBySNI(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeAcmeStore(t, dir, "letsencrypt", map[string][]string{
|
||||
"mail.example.test": nil,
|
||||
"webmail.example.test": nil,
|
||||
})
|
||||
|
||||
watcher, err := NewAcmeCertWatcher(dir, "acme.json", "mail.example.test")
|
||||
if err != nil {
|
||||
t.Fatalf("NewAcmeCertWatcher: %v", err)
|
||||
}
|
||||
|
||||
cert, err := watcher.GetCertificate(&tls.ClientHelloInfo{ServerName: "webmail.example.test"})
|
||||
if err != nil || cert == nil {
|
||||
t.Fatalf("GetCertificate: cert=%v err=%v", cert, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The resolver name varies per Coolify installation, so the parser must not
|
||||
// depend on it (PLAN.md section 8.2, trap 3).
|
||||
func TestAcmeCertWatcher_HandlesAnyResolverName(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeAcmeStore(t, dir, "some-custom-resolver-name", map[string][]string{"mail.example.test": nil})
|
||||
|
||||
watcher, err := NewAcmeCertWatcher(dir, "acme.json", "mail.example.test")
|
||||
if err != nil {
|
||||
t.Fatalf("NewAcmeCertWatcher: %v", err)
|
||||
}
|
||||
if !watcher.HasCertificateFor("mail.example.test") {
|
||||
t.Error("certificate was not found under a non-default resolver name")
|
||||
}
|
||||
}
|
||||
|
||||
// A SAN entry must be served too, not just the main domain.
|
||||
func TestAcmeCertWatcher_IndexesSubjectAlternativeNames(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeAcmeStore(t, dir, "letsencrypt", map[string][]string{
|
||||
"example.test": {"mta-sts.example.test", "autoconfig.example.test"},
|
||||
})
|
||||
|
||||
watcher, _ := NewAcmeCertWatcher(dir, "acme.json", "example.test")
|
||||
|
||||
for _, name := range []string{"example.test", "mta-sts.example.test", "autoconfig.example.test"} {
|
||||
if !watcher.HasCertificateFor(name) {
|
||||
t.Errorf("no certificate indexed for %s", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Traefik replaces the file by rename. A watcher bound to the old inode would
|
||||
// never see this; polling the path does (PLAN.md section 8.2, trap 1).
|
||||
func TestAcmeCertWatcher_PicksUpAtomicRename(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeAcmeStore(t, dir, "letsencrypt", map[string][]string{"mail.example.test": nil})
|
||||
|
||||
watcher, err := NewAcmeCertWatcher(dir, "acme.json", "mail.example.test")
|
||||
if err != nil {
|
||||
t.Fatalf("NewAcmeCertWatcher: %v", err)
|
||||
}
|
||||
if watcher.HasCertificateFor("new.example.test") {
|
||||
t.Fatal("the new host is present before the renewal")
|
||||
}
|
||||
|
||||
// Simulate Traefik: write a temporary file, then rename it over the old one.
|
||||
staging := t.TempDir()
|
||||
writeAcmeStore(t, staging, "letsencrypt", map[string][]string{
|
||||
"mail.example.test": nil,
|
||||
"new.example.test": nil,
|
||||
})
|
||||
if err := os.Rename(filepath.Join(staging, "acme.json"), filepath.Join(dir, "acme.json")); err != nil {
|
||||
t.Fatalf("rename: %v", err)
|
||||
}
|
||||
|
||||
if err := watcher.reload(); err != nil {
|
||||
t.Fatalf("reload after rename: %v", err)
|
||||
}
|
||||
if !watcher.HasCertificateFor("new.example.test") {
|
||||
t.Error("the renewed store was not picked up after the atomic rename")
|
||||
}
|
||||
}
|
||||
|
||||
// An unreadable or truncated file must never replace a working store with an
|
||||
// empty one (PLAN.md section 8.4).
|
||||
func TestAcmeCertWatcher_KeepsPreviousStoreOnParseFailure(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := writeAcmeStore(t, dir, "letsencrypt", map[string][]string{"mail.example.test": nil})
|
||||
|
||||
watcher, _ := NewAcmeCertWatcher(dir, "acme.json", "mail.example.test")
|
||||
|
||||
if err := os.WriteFile(path, []byte("{ truncated"), 0o600); err != nil {
|
||||
t.Fatalf("write truncated file: %v", err)
|
||||
}
|
||||
if err := watcher.reload(); err == nil {
|
||||
t.Fatal("expected reload to report the parse failure")
|
||||
}
|
||||
|
||||
if !watcher.HasCertificateFor("mail.example.test") {
|
||||
t.Error("a truncated file wiped out the certificates already loaded")
|
||||
}
|
||||
}
|
||||
|
||||
// A store with no certificate for the mail host is a deployment error: no
|
||||
// router was declared for it, so Traefik never requested one.
|
||||
func TestAcmeCertWatcher_ReportsMissingMailHost(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeAcmeStore(t, dir, "letsencrypt", map[string][]string{"webmail.example.test": nil})
|
||||
|
||||
watcher, _ := NewAcmeCertWatcher(dir, "acme.json", "mail.example.test")
|
||||
|
||||
if watcher.HasCertificateFor("mail.example.test") {
|
||||
t.Error("HasCertificateFor must report the missing mail host so the preflight can fail")
|
||||
}
|
||||
}
|
||||
|
||||
// An unknown SNI falls back to the mail host certificate and is counted.
|
||||
func TestAcmeCertWatcher_UnknownSniFallsBackAndCounts(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeAcmeStore(t, dir, "letsencrypt", map[string][]string{"mail.example.test": nil})
|
||||
|
||||
watcher, _ := NewAcmeCertWatcher(dir, "acme.json", "mail.example.test")
|
||||
|
||||
cert, err := watcher.GetCertificate(&tls.ClientHelloInfo{ServerName: "stranger.invalid"})
|
||||
if err != nil || cert == nil {
|
||||
t.Fatalf("expected the default certificate, got cert=%v err=%v", cert, err)
|
||||
}
|
||||
if watcher.UnknownSNICount() != 1 {
|
||||
t.Errorf("UnknownSNICount = %d, want 1", watcher.UnknownSNICount())
|
||||
}
|
||||
}
|
||||
|
||||
// A wildcard certificate must cover the hosts under it.
|
||||
func TestAcmeCertWatcher_MatchesWildcard(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeAcmeStore(t, dir, "letsencrypt", map[string][]string{"*.example.test": nil})
|
||||
|
||||
watcher, _ := NewAcmeCertWatcher(dir, "acme.json", "*.example.test")
|
||||
|
||||
if _, err := watcher.GetCertificate(&tls.ClientHelloInfo{ServerName: "mail.example.test"}); err != nil {
|
||||
t.Errorf("wildcard certificate did not cover mail.example.test: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A missing file at startup is fatal on purpose: the operator must know
|
||||
// immediately, not ninety days later.
|
||||
func TestAcmeCertWatcher_FailsWhenStoreMissing(t *testing.T) {
|
||||
if _, err := NewAcmeCertWatcher(t.TempDir(), "acme.json", "mail.example.test"); err == nil {
|
||||
t.Fatal("expected construction to fail without an acme.json")
|
||||
}
|
||||
}
|
||||
|
||||
// An identical file must not cause a pointless swap; LastReload only advances
|
||||
// when the content actually changed.
|
||||
func TestAcmeCertWatcher_SkipsReloadWhenContentUnchanged(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeAcmeStore(t, dir, "letsencrypt", map[string][]string{"mail.example.test": nil})
|
||||
|
||||
watcher, _ := NewAcmeCertWatcher(dir, "acme.json", "mail.example.test")
|
||||
first := watcher.LastReload()
|
||||
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
if err := watcher.reload(); err != nil {
|
||||
t.Fatalf("reload: %v", err)
|
||||
}
|
||||
|
||||
if !watcher.LastReload().Equal(first) {
|
||||
t.Error("an unchanged file triggered a reload")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user