v0.23.0 color, presets, certificates made at startup, cli.go, sink flush

internal/tlsx/pki.go

@@ -0,0 +1,231 @@
+package tlsx
+
+import (
+	"bytes"
+	"cmp"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/base64"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+	"unicode"
+)
+
+// Certificates are ECDSA P-256, for lw tls and for listeners without files.
+// A CA signs leaves only; a leaf lasts at most 397 days, the longest browsers
+// accept, and never past its issuer.
+const (
+	CAValidity   = 10 * 365 * 24 * time.Hour
+	LeafValidity = 397 * 24 * time.Hour
+	pinPrefix    = "sha256//"
+)
+
+// processKey is the key of every generated listener certificate: reloads
+// reissue certificates and keep it, so a pin holds until lw restarts.
+var processKey = sync.OnceValues(NewKey)
+
+func NewKey() (*ecdsa.PrivateKey, error) { return ecdsa.GenerateKey(elliptic.P256(), rand.Reader) }
+
+// NewCA makes a self-signed certificate authority that signs leaves only
+func NewCA(name string, validity time.Duration) (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	key, err := NewKey()
+	if err != nil {
+		return nil, nil, err
+	}
+	t, err := template(name, validity)
+	if err != nil {
+		return nil, nil, err
+	}
+	t.IsCA, t.MaxPathLenZero = true, true
+	t.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign
+	cert, err := sign(t, t, &key.PublicKey, key)
+	return cert, key, err
+}
+
+// Leaf describes a certificate to issue. Hosts become DNS or IP SANs, which
+// hostname verification reads; Name is the subject CN, which mtls reads.
+type Leaf struct {
+	Name           string
+	Hosts          []string
+	Server, Client bool
+	Validity       time.Duration
+}
+
+// Issue signs a certificate for pub; a nil issuer makes it self-signed.
+func (l Leaf) Issue(pub crypto.PublicKey, issuer *x509.Certificate, issuerKey crypto.Signer) (*x509.Certificate, error) {
+	t, err := template(l.Name, l.Validity)
+	if err != nil {
+		return nil, err
+	}
+	t.KeyUsage = x509.KeyUsageDigitalSignature
+	if l.Server {
+		t.ExtKeyUsage = append(t.ExtKeyUsage, x509.ExtKeyUsageServerAuth)
+	}
+	if l.Client {
+		t.ExtKeyUsage = append(t.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
+	}
+	for _, h := range l.Hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			t.IPAddresses = append(t.IPAddresses, ip)
+		} else {
+			t.DNSNames = append(t.DNSNames, h)
+		}
+	}
+	if issuer == nil {
+		return sign(t, t, pub, issuerKey)
+	}
+	if !time.Now().Before(issuer.NotAfter) {
+		return nil, fmt.Errorf("tls: issuer %q expired on %s", issuer.Subject.CommonName, issuer.NotAfter.UTC().Format(time.DateOnly))
+	}
+	if t.NotAfter.After(issuer.NotAfter) {
+		t.NotAfter = issuer.NotAfter
+	}
+	return sign(t, issuer, pub, issuerKey)
+}
+
+func template(name string, validity time.Duration) (*x509.Certificate, error) {
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, err
+	}
+	now := time.Now()
+	return &x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: name},
+		NotBefore:             now.Add(-5 * time.Minute), // peers' clocks drift
+		NotAfter:              now.Add(validity),
+		BasicConstraintsValid: true,
+	}, nil
+}
+
+func sign(t, parent *x509.Certificate, pub crypto.PublicKey, key crypto.Signer) (*x509.Certificate, error) {
+	der, err := x509.CreateCertificate(rand.Reader, t, parent, pub, key)
+	if err != nil {
+		return nil, fmt.Errorf("tls: sign certificate: %w", err)
+	}
+	return x509.ParseCertificate(der)
+}
+
+// LoadIssuer reads a CA certificate and its key, as lw tls ca writes them
+func LoadIssuer(certFile, keyFile string) (*x509.Certificate, crypto.Signer, error) {
+	pair, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, nil, fmt.Errorf("tls: load issuer: %w", err)
+	}
+	ca, err := x509.ParseCertificate(pair.Certificate[0]) // Leaf is nil under GODEBUG=x509keypairleaf=0
+	if err != nil {
+		return nil, nil, fmt.Errorf("tls: load issuer: %w", err)
+	}
+	signer, ok := pair.PrivateKey.(crypto.Signer)
+	if !ok || !ca.IsCA || ca.KeyUsage&x509.KeyUsageCertSign == 0 {
+		return nil, nil, fmt.Errorf("tls: %s is no certificate authority", certFile)
+	}
+	return ca, signer, nil
+}
+
+// generated issues a listener certificate on the process key, self-signed
+// or from the issuer files, for hosts and the names this machine answers to.
+func generated(issuerCert, issuerKey, host string, hosts []string) (tls.Certificate, error) {
+	key, err := processKey()
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	var issuer *x509.Certificate
+	var signer crypto.Signer = key
+	if issuerCert != "" {
+		if issuer, signer, err = LoadIssuer(issuerCert, issuerKey); err != nil {
+			return tls.Certificate{}, err
+		}
+	}
+	name, _ := os.Hostname()
+	all := slices.Clone(hosts)
+	if ip := net.ParseIP(host); host != "" && (ip == nil || !ip.IsUnspecified()) {
+		all = append(all, host)
+	}
+	if !strings.ContainsFunc(name, func(r rune) bool { return r > unicode.MaxASCII }) {
+		all = append(all, name) // a DNS SAN is ASCII; the CN keeps any name
+	}
+	all = append(all, "localhost", "127.0.0.1", "::1")
+	slices.Sort(all)
+	all = slices.Compact(slices.DeleteFunc(all, func(h string) bool { return h == "" }))
+	leaf, err := Leaf{Name: cmp.Or(name, "logwisp"), Hosts: all, Server: true, Validity: LeafValidity}.Issue(&key.PublicKey, issuer, signer)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	chain := [][]byte{leaf.Raw}
+	if issuer != nil {
+		chain = append(chain, issuer.Raw) // an intermediate needs it to chain to its root
+	}
+	return tls.Certificate{Certificate: chain, PrivateKey: key, Leaf: leaf}, nil
+}
+
+// PinSHA256 is the curl --pinnedpubkey form of a certificate's public key
+func PinSHA256(c *x509.Certificate) string { return pin(c.RawSubjectPublicKeyInfo) }
+
+func pinOfKey(k *ecdsa.PrivateKey) string {
+	der, _ := x509.MarshalPKIXPublicKey(&k.PublicKey) // P-256 always marshals
+	return pin(der)
+}
+
+func pin(spki []byte) string {
+	h := sha256.Sum256(spki)
+	return pinPrefix + base64.StdEncoding.EncodeToString(h[:])
+}
+
+// verifyPins replaces chain verification: the server's key must hash to one
+// of the ';'-separated pins. Its name and validity are not checked. Go skips
+// it on a resumed session, and no dialer keeps a ClientSessionCache.
+func verifyPins(s string) (func([][]byte, [][]*x509.Certificate) error, error) {
+	var pins [][]byte
+	for p := range strings.SplitSeq(s, ";") {
+		b64, ok := strings.CutPrefix(strings.TrimSpace(p), pinPrefix)
+		h, err := base64.StdEncoding.DecodeString(b64)
+		if !ok || err != nil || len(h) != sha256.Size {
+			return nil, fmt.Errorf("tls: pin_sha256 %q: want %sBASE64 of a SHA-256, ';' between several", p, pinPrefix)
+		}
+		pins = append(pins, h)
+	}
+	return func(raw [][]byte, _ [][]*x509.Certificate) error {
+		if len(raw) == 0 {
+			return errors.New("tls: server sent no certificate")
+		}
+		c, err := x509.ParseCertificate(raw[0])
+		if err != nil {
+			return err
+		}
+		h := sha256.Sum256(c.RawSubjectPublicKeyInfo)
+		if slices.ContainsFunc(pins, func(p []byte) bool { return bytes.Equal(p, h[:]) }) {
+			return nil
+		}
+		// The type of a failed chain check: callers treat both as a refusal
+		return &tls.CertificateVerificationError{UnverifiedCertificates: []*x509.Certificate{c},
+			Err: fmt.Errorf("server key %s matches no tls.pin_sha256", PinSHA256(c))}
+	}, nil
+}
+
+// EncodeCert and EncodeKey render PEM, the key as PKCS #8
+func EncodeCert(c *x509.Certificate) []byte {
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.Raw})
+}
+
+func EncodeKey(k crypto.PrivateKey) ([]byte, error) {
+	der, err := x509.MarshalPKCS8PrivateKey(k)
+	if err != nil {
+		return nil, err
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil
+}

internal/tlsx/tlsx.go

@@ -26,16 +26,36 @@ const (
 	expiryWindow = 30 * 24 * time.Hour
 )
 
-// Server builds the *tls.Config for listener plugins
-// (tcp/http sinks, tcp_chain/http_chain sources). Returns (nil, nil) when disabled.
-func Server(o *config.TLSOptions) (*tls.Config, error) {
+// Server builds the *tls.Config for listener plugins (tcp/http sinks,
+// tcp_chain/http_chain sources) from cert_file and key_file, or issues one on
+// the process key: self-signed or from the issuer files, for host, hosts and
+// this machine's names. Returns (nil, nil) when disabled.
+func Server(o *config.TLSOptions, host string) (*tls.Config, error) {
 	if o == nil || !o.Enabled {
 		return nil, nil
 	}
-	if o.CertFile == "" || o.KeyFile == "" {
-		return nil, fmt.Errorf("tls: cert_file and key_file are required for listeners")
+	files, issuer := o.CertFile != "" || o.KeyFile != "", o.IssuerCertFile != "" || o.IssuerKeyFile != ""
+	switch {
+	case o.PinSHA256 != "":
+		return nil, fmt.Errorf("tls: pin_sha256 applies to dialers")
+	case files && (issuer || o.SelfSigned) || issuer && o.SelfSigned:
+		return nil, fmt.Errorf("tls: set one of cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file")
+	case len(o.Hosts) > 0 && !issuer && !o.SelfSigned:
+		return nil, fmt.Errorf("tls: hosts applies to self_signed and issuer certificates")
+	case files && (o.CertFile == "" || o.KeyFile == ""):
+		return nil, fmt.Errorf("tls: cert_file and key_file must be set together")
+	case issuer && (o.IssuerCertFile == "" || o.IssuerKeyFile == ""):
+		return nil, fmt.Errorf("tls: issuer_cert_file and issuer_key_file must be set together")
+	case !files && !issuer && !o.SelfSigned:
+		return nil, fmt.Errorf("tls: listeners need cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file")
+	}
+	var cert tls.Certificate
+	var err error
+	if files {
+		cert, err = tls.LoadX509KeyPair(o.CertFile, o.KeyFile)
+	} else {
+		cert, err = generated(o.IssuerCertFile, o.IssuerKeyFile, host, o.Hosts)
 	}
-	cert, err := tls.LoadX509KeyPair(o.CertFile, o.KeyFile)
 	if err != nil {
 		return nil, fmt.Errorf("tls: load keypair: %w", err)
 	}
@@ -64,11 +84,18 @@ func Server(o *config.TLSOptions) (*tls.Config, error) {
 // Client builds the *tls.Config for dialer plugins (tcp_chain/http_chain
 // sinks). host seeds ServerName when no override is set; Go verifies IP SANs
 // when host is an address, without its zone, which names a local interface
-// and no certificate carries. Returns (nil, nil) when disabled.
+// and no certificate carries. pin_sha256 replaces chain and name checks.
+// Returns (nil, nil) when disabled.
 func Client(o *config.TLSOptions, host string) (*tls.Config, error) {
 	if o == nil || !o.Enabled {
 		return nil, nil
 	}
+	switch {
+	case o.SelfSigned || o.IssuerCertFile != "" || o.IssuerKeyFile != "" || len(o.Hosts) > 0:
+		return nil, fmt.Errorf("tls: self_signed, issuer_cert_file, issuer_key_file and hosts apply to listeners")
+	case o.PinSHA256 != "" && (o.CAFile != "" || o.InsecureSkipVerify):
+		return nil, fmt.Errorf("tls: pin_sha256 replaces ca_file and insecure_skip_verify: set one")
+	}
 	mv, err := minVersion(o.MinVersion)
 	if err != nil {
 		return nil, err
@@ -81,6 +108,12 @@ func Client(o *config.TLSOptions, host string) (*tls.Config, error) {
 	if cfg.ServerName == "" {
 		cfg.ServerName, _, _ = strings.Cut(host, "%")
 	}
+	if o.PinSHA256 != "" {
+		if cfg.VerifyPeerCertificate, err = verifyPins(o.PinSHA256); err != nil {
+			return nil, err
+		}
+		cfg.InsecureSkipVerify = true // the pin check above replaces it
+	}
 	if o.CAFile != "" {
 		pool, err := loadPool(o.CAFile)
 		if err != nil {
@@ -149,13 +182,22 @@ func LogWarnings(l *log.Logger, component, id string, o *config.TLSOptions, list
 	if o == nil || !o.Enabled {
 		return
 	}
+	if key, err := processKey(); listener && o.SelfSigned && err == nil {
+		// Without a CA, a dialer verifies this listener by its pin only
+		l.Warn("msg", "Serving a self-signed certificate; it changes when lw restarts",
+			"component", component,
+			"instance_id", id,
+			"pin_sha256", pinOfKey(key),
+			"hint", "dialers set tls.pin_sha256; for a stable identity use issuer_cert_file or cert_file")
+	}
 	var warnings []string
 	if !listener && o.InsecureSkipVerify {
 		warnings = append(warnings, "tls.insecure_skip_verify disables server verification: any peer can impersonate the server")
 	}
-	files := []struct{ key, path string }{{"cert_file", o.CertFile}, {"ca_file", o.CAFile}}
+	type file struct{ key, path string }
+	files := []file{{"cert_file", o.CertFile}, {"ca_file", o.CAFile}}
 	if listener {
-		files[1] = struct{ key, path string }{"client_ca_file", o.ClientCAFile}
+		files = []file{{"cert_file", o.CertFile}, {"client_ca_file", o.ClientCAFile}, {"issuer_cert_file", o.IssuerCertFile}}
 	}
 	now := time.Now()
 	for _, f := range files {
@@ -165,8 +207,10 @@ func LogWarnings(l *log.Logger, component, id string, o *config.TLSOptions, list
 			}
 		}
 	}
-	if w := SecretFileWarning("tls.key_file", o.KeyFile); w != "" {
-		warnings = append(warnings, w)
+	for _, key := range []file{{"tls.key_file", o.KeyFile}, {"tls.issuer_key_file", o.IssuerKeyFile}} {
+		if w := SecretFileWarning(key.key, key.path); w != "" {
+			warnings = append(warnings, w)
+		}
 	}
 	for _, w := range warnings {
 		l.Warn("msg", w, "component", component, "instance_id", id)

internal/tlsx/tlsx_test.go

@@ -3,11 +3,15 @@ package tlsx
 import (
 	"crypto/x509"
 	"crypto/x509/pkix"
+	"net"
+	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/lixenwraith/logwisp/internal/config"
+	"github.com/lixenwraith/logwisp/internal/testutil"
 )
 
 // A dialer verifies an IPv6 target by its address: the zone names a local
@@ -48,3 +52,104 @@ func TestExpiryWarning(t *testing.T) {
 		}
 	}
 }
+
+// A self-signed listener is verified by the pin of the process key, which a
+// reload's reissued certificate keeps; any other pin, or none, fails.
+func TestSelfSignedListenerIsVerifiedByItsPin(t *testing.T) {
+	o := &config.TLSOptions{Enabled: true, SelfSigned: true, Hosts: []string{"agg.example"}}
+	first, err := Server(o, "0.0.0.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	reloaded, err := Server(o, "0.0.0.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	leaf := reloaded.Certificates[0].Leaf
+	pin := PinSHA256(leaf)
+	if PinSHA256(first.Certificates[0].Leaf) != pin || !slices.Contains(leaf.DNSNames, "agg.example") ||
+		!slices.Contains(leaf.DNSNames, "localhost") || slices.ContainsFunc(leaf.IPAddresses, net.IP.IsUnspecified) {
+		t.Fatalf("pin %s, names %v %v", pin, leaf.DNSNames, leaf.IPAddresses)
+	}
+	other := "sha256//" + strings.Repeat("A", 43) + "="
+	for pins, want := range map[string]string{other + "; " + pin: "", other: "matches no tls.pin_sha256", "": "certificate"} {
+		client, err := Client(&config.TLSOptions{Enabled: true, PinSHA256: pins}, "agg.example")
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = testutil.Handshake(t, reloaded, client)
+		if want == "" && err != nil || want != "" && (err == nil || !strings.Contains(err.Error(), want)) {
+			t.Errorf("pins %q: %v, want %q", pins, err, want)
+		}
+	}
+}
+
+// An issuer-signed listener chains to the CA file, for its host, and expires
+// with its issuer; a certificate that is no CA cannot issue.
+func TestIssuedListenerCertificateChainsToTheIssuer(t *testing.T) {
+	dir := t.TempDir()
+	ca, key, err := NewCA("test CA", time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+	keyPEM, err := EncodeKey(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	caFile, keyFile := filepath.Join(dir, "ca.crt"), filepath.Join(dir, "ca.key")
+	testutil.WriteFile(t, caFile, string(EncodeCert(ca)))
+	testutil.WriteFile(t, keyFile, string(keyPEM))
+	server, err := Server(&config.TLSOptions{Enabled: true, IssuerCertFile: caFile, IssuerKeyFile: keyFile}, "127.0.0.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if leaf := server.Certificates[0].Leaf; !leaf.NotAfter.Equal(ca.NotAfter) {
+		t.Fatalf("leaf expires %s, issuer %s", leaf.NotAfter, ca.NotAfter)
+	}
+	client, err := Client(&config.TLSOptions{Enabled: true, CAFile: caFile}, "127.0.0.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := testutil.Handshake(t, server, client); err != nil {
+		t.Fatalf("issued chain: %v", err)
+	}
+	leafFile := filepath.Join(dir, "leaf.crt")
+	testutil.WriteFile(t, leafFile, string(EncodeCert(server.Certificates[0].Leaf)))
+	if _, err := Server(&config.TLSOptions{Enabled: true, IssuerCertFile: leafFile, IssuerKeyFile: keyFile}, ""); err == nil {
+		t.Fatal("a leaf issued certificates")
+	}
+}
+
+// Each role takes one source of trust and one certificate source; keys of the
+// other role fail instead of doing nothing.
+func TestTLSOptionConflicts(t *testing.T) {
+	pin := "sha256//" + strings.Repeat("A", 43) + "="
+	for _, c := range []struct {
+		o      config.TLSOptions
+		server bool
+		want   string
+	}{
+		{config.TLSOptions{SelfSigned: true, CertFile: "c", KeyFile: "k"}, true, "set one of"},
+		{config.TLSOptions{SelfSigned: true, IssuerCertFile: "c"}, true, "set one of"},
+		{config.TLSOptions{IssuerCertFile: "c"}, true, "must be set together"},
+		{config.TLSOptions{CertFile: "c"}, true, "must be set together"},
+		{config.TLSOptions{Hosts: []string{"a"}, CertFile: "c", KeyFile: "k"}, true, "hosts applies to"},
+		{config.TLSOptions{SelfSigned: true, PinSHA256: pin}, true, "applies to dialers"},
+		{config.TLSOptions{}, true, "listeners need"},
+		{config.TLSOptions{SelfSigned: true}, false, "apply to listeners"},
+		{config.TLSOptions{PinSHA256: pin, CAFile: "ca"}, false, "set one"},
+		{config.TLSOptions{PinSHA256: pin, InsecureSkipVerify: true}, false, "set one"},
+		{config.TLSOptions{PinSHA256: "sha256//short"}, false, "want sha256//BASE64"},
+	} {
+		c.o.Enabled = true
+		var err error
+		if c.server {
+			_, err = Server(&c.o, "")
+		} else {
+			_, err = Client(&c.o, "host")
+		}
+		if err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("%+v: %v, want %q", c.o, err, c.want)
+		}
+	}
+}