Trilltechnician/incredigo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rotate: promote k8s + mongo to LIVE-VM via real-target POCs

Browse Source 
Ran live cutover POCs in the sandbox VM against the real target software and
promoted both drivers from MOCK-ONLY to LIVE-VM in the proof table (data, not
driver behaviour):

- k8s: proven against real k3s v1.35 kube-apiserver + token controller
  (lab-provision-k8s.sh). The real apiserver serves a self-signed cert, so the
  driver gained CA-pinned TLS — blob params ca= (base64 PEM, pinned as the only
  trusted root) and insecure=1 (lab-only escape hatch), routed through a new
  clientFor(cr). Neither is test-awareness; a real deployment configures the same
  CA. Added TestK8sTLSTrust covering CA-pinned / insecure / untrusted-rejection
  and the blob round-trip.
- mongo: proven against real mongod 8.0 (lab-provision-mongo.sh).

npm stays MOCK-ONLY by decision (registry.npmjs.org not self-hostable; real
create-token requires the account password in the body) — documented as a
contract gap rather than faking a LIVE-VM. gcp/twilio/flyio remain MOCK-ONLY.

Live POCs surfaced real-target-only behaviour now recorded in the docs: the
apiserver's ~10s successful-auth cache (old token kept working ~7s after Secret
deletion) and real mongosh's stdout prompt. Full suite 104 green, -race clean.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
leetcrypt
2026-06-18 21:50:21 -07:00
parent 35f19c007a
commit 9109da7ae4
5 changed files with 160 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/rotate/k8s.go
+58 -17
View File
@@ -3,6 +3,8 @@ package rotate
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"fmt"
@@ -32,6 +34,10 @@ import (
// - sa= the ServiceAccount name (the new Secret is annotated to it).
// - secret= the token Secret's name — required to DELETE the old token.
// - token= the bearer token value (rotated).
// - ca= OPTIONAL base64(std) of the cluster CA PEM bundle. A real apiserver
// serves a self-signed cert, so the driver pins this CA as the only
// trusted root for the TLS handshake. Omit for a loopback emulator.
// - insecure=1 OPTIONAL escape hatch: skip TLS verification entirely (lab only).
//
// AUTH: every call authenticates with a token Bearer header against the apiserver. The
// rotating SA needs RBAC to create/delete Secrets in its namespace. The new token is
@@ -53,20 +59,44 @@ func (k *K8s) Name() string { return "k8s" }
// Detect claims credentials tagged Source == "k8s".
func (k *K8s) Detect(c discover.Credential) bool { return c.Source == "k8s" }
func (k *K8s) client() *http.Client {
// clientFor returns the HTTP client for a given credential. An injected client
// (tests / explicit CA config) always wins. Otherwise it builds a 15s client whose
// TLS trust is pinned to the credential's ca= bundle (a real apiserver's self-signed
// cert), or with verification disabled when insecure=1 is set (lab only).
func (k *K8s) clientFor(cr k8sCred) (*http.Client, error) {
if k.HTTPClient != nil {
return k.HTTPClient
return k.HTTPClient, nil
}
return &http.Client{Timeout: 15 * time.Second}
tlsCfg := &tls.Config{MinVersion: tls.VersionTLS12}
switch {
case cr.insecure:
tlsCfg.InsecureSkipVerify = true
case cr.ca != "":
pem, err := base64.StdEncoding.DecodeString(cr.ca)
if err != nil {
return nil, fmt.Errorf("k8s: decode ca bundle: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, fmt.Errorf("k8s: ca bundle contained no usable certificates")
}
tlsCfg.RootCAs = pool
}
return &http.Client{
Timeout: 15 * time.Second,
Transport: &http.Transport{TLSClientConfig: tlsCfg},
}, nil
}
// k8sCred is the parsed credential blob. None of its fields are ever logged.
type k8sCred struct {
base string // scheme://host:port
ns string
sa string
secret string // token Secret name
token string
base string // scheme://host:port
ns string
sa string
secret string // token Secret name
token string
ca string // base64(std) cluster CA PEM bundle (optional)
insecure bool // skip TLS verification (lab only)
}
func parseK8sCred(v *vault.Vault, h *vault.Handle) (k8sCred, error) {
@@ -89,7 +119,7 @@ func parseK8sCred(v *vault.Vault, h *vault.Handle) (k8sCred, error) {
return k8sCred{}, fmt.Errorf("k8s: secret has no apiserver host")
}
q := u.Query()
c := k8sCred{base: scheme + "://" + u.Host, ns: q.Get("ns"), sa: q.Get("sa"), secret: q.Get("secret"), token: q.Get("token")}
c := k8sCred{base: scheme + "://" + u.Host, ns: q.Get("ns"), sa: q.Get("sa"), secret: q.Get("secret"), token: q.Get("token"), ca: q.Get("ca"), insecure: q.Get("insecure") == "1"}
if c.ns == "" || c.sa == "" {
return k8sCred{}, fmt.Errorf("k8s: secret missing namespace or serviceaccount")
}
@@ -110,6 +140,12 @@ func (c k8sCred) build() string {
q.Set("secret", c.secret)
}
q.Set("token", c.token)
if c.ca != "" {
q.Set("ca", c.ca)
}
if c.insecure {
q.Set("insecure", "1")
}
u.RawQuery = q.Encode()
return u.String()
}
@@ -132,8 +168,9 @@ type k8sSecretObj struct {
Data map[string]string `json:"data,omitempty"`
}
// do issues an authenticated apiserver request and returns the body for 2xx responses.
func (k *K8s) do(ctx context.Context, method, urlStr, token string, body []byte) ([]byte, int, error) {
// do issues an authenticated apiserver request and returns the body. The credential
// carries the TLS trust config (ca=/insecure=) used to build the client.
func (k *K8s) do(ctx context.Context, cr k8sCred, method, urlStr string, body []byte) ([]byte, int, error) {
var r io.Reader
if body != nil {
r = bytes.NewReader(body)
@@ -142,11 +179,15 @@ func (k *K8s) do(ctx context.Context, method, urlStr, token string, body []byte)
if err != nil {
return nil, 0, err
}
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Authorization", "Bearer "+cr.token)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := k.client().Do(req)
cl, err := k.clientFor(cr)
if err != nil {
return nil, 0, err
}
resp, err := cl.Do(req)
if err != nil {
return nil, 0, err
}
@@ -181,7 +222,7 @@ func (k *K8s) Rotate(ctx context.Context, c discover.Credential, v *vault.Vault)
obj.Metadata.Annotations = map[string]string{"kubernetes.io/service-account.name": cr.sa}
body, _ := json.Marshal(obj)
_, code, err := k.do(ctx, http.MethodPost, cr.secretsURL(""), cr.token, body)
_, code, err := k.do(ctx, cr, http.MethodPost, cr.secretsURL(""), body)
if err != nil {
return nil, fmt.Errorf("k8s: create token secret: %w", err)
}
@@ -203,7 +244,7 @@ func (k *K8s) Rotate(ctx context.Context, c discover.Credential, v *vault.Vault)
// awaitToken polls the new Secret until its .data.token is present, decoding it.
func (k *K8s) awaitToken(ctx context.Context, cr k8sCred, name string) (string, error) {
for attempt := 0; attempt < 10; attempt++ {
rb, code, err := k.do(ctx, http.MethodGet, cr.secretsURL(name), cr.token, nil)
rb, code, err := k.do(ctx, cr, http.MethodGet, cr.secretsURL(name), nil)
if err != nil {
return "", fmt.Errorf("k8s: read token secret: %w", err)
}
@@ -236,7 +277,7 @@ func (k *K8s) Verify(ctx context.Context, newSecret *vault.Handle, v *vault.Vaul
return err
}
saURL := cr.base + "/api/v1/namespaces/" + url.PathEscape(cr.ns) + "/serviceaccounts/" + url.PathEscape(cr.sa)
_, code, err := k.do(ctx, http.MethodGet, saURL, cr.token, nil)
_, code, err := k.do(ctx, cr, http.MethodGet, saURL, nil)
if err != nil {
return fmt.Errorf("k8s verify: %w", err)
}
@@ -257,7 +298,7 @@ func (k *K8s) RevokeOld(ctx context.Context, c discover.Credential, v *vault.Vau
if cr.secret == "" {
return fmt.Errorf("k8s revoke: old token has no recorded secret name — delete it manually (guided)")
}
_, code, err := k.do(ctx, http.MethodDelete, cr.secretsURL(cr.secret), cr.token, nil)
_, code, err := k.do(ctx, cr, http.MethodDelete, cr.secretsURL(cr.secret), nil)
if err != nil {
return fmt.Errorf("k8s revoke: %w", err)
}
internal/rotate/k8s_test.go
+52
View File
@@ -6,8 +6,10 @@ import (
"encoding/base64"
"encoding/hex"
"encoding/json"
"encoding/pem"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"sync"
"testing"
@@ -218,6 +220,56 @@ func TestK8sRevokeRequiresSecretName(t *testing.T) {
}
}
// TestK8sTLSTrust proves the ca=/insecure= plumbing actually drives the TLS
// handshake when no client is injected (the real-apiserver path). The httptest
// server serves a self-signed cert, so a default client rejects it; pinning its CA
// or setting insecure=1 must make the same request succeed.
func TestK8sTLSTrust(t *testing.T) {
v := vault.New()
defer v.Purge()
ctx := context.Background()
const ns, sa, tok = "labns", "labsa", "eyJtls0123"
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/namespaces/"+ns+"/serviceaccounts/", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]any{"metadata": map[string]any{"name": sa}})
})
srv := httptest.NewTLSServer(mux)
defer srv.Close()
host := strings.TrimPrefix(srv.URL, "https://")
caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
caB64 := base64.StdEncoding.EncodeToString(caPEM)
verifyWith := func(blob string) error {
k := &K8s{} // NO injected client — exercises clientFor's real TLS path
h := v.Store([]byte(blob))
return k.Verify(ctx, h, v)
}
base := "k8s://" + host + "/?ns=" + ns + "&sa=" + sa + "&token=" + tok
if err := verifyWith(base); err == nil {
t.Error("untrusted self-signed cert should fail without ca=/insecure=")
}
if err := verifyWith(base + "&insecure=1"); err != nil {
t.Errorf("insecure=1 should skip verification: %v", err)
}
if err := verifyWith(base + "&ca=" + url.QueryEscape(caB64)); err != nil {
t.Errorf("pinned ca= should trust the server cert: %v", err)
}
// Blob round-trip: ca=/insecure= survive build()->parse().
cr := k8sCred{base: "https://" + host, ns: ns, sa: sa, token: tok, ca: caB64, insecure: true}
got, err := parseK8sCred(v, v.Store([]byte(cr.build())))
if err != nil {
t.Fatalf("round-trip parse: %v", err)
}
if got.ca != caB64 || !got.insecure {
t.Errorf("ca/insecure did not round-trip: ca-match=%v insecure=%v", got.ca == caB64, got.insecure)
}
}
func TestK8sRejectsBadSecret(t *testing.T) {
v := vault.New()
defer v.Purge()
internal/rotate/proofs.go
+2 -2
View File
@@ -54,6 +54,8 @@ var proofLevels = map[string]ProofLevel{
"wireguard": ProofLiveVM, // real wg (lab-provision-wg.sh)
"gitea": ProofLiveVM, // real Gitea (lab-provision-gitea.sh)
"appsecret": ProofLiveVM, // real local files (lab-provision-appsec.sh)
"mongo": ProofLiveVM, // real mongod 8.0 (lab-provision-mongo.sh)
"k8s": ProofLiveVM, // real k3s v1.35 (lab-provision-k8s.sh)
// proven only against an emulator / mock:
"aws": ProofMockOnly, // moto mock AWS in VM; real AWS never hit
@@ -63,12 +65,10 @@ var proofLevels = map[string]ProofLevel{
"cloudflare": ProofMockOnly, // httptest emulator; no self-host
"ghactions": ProofMockOnly, // httptest emulator; no self-host
"gitlab": ProofMockOnly, // httptest emulator; GitLab CE too heavy for the VM
"mongo": ProofMockOnly, // fake-mongosh stub; real-MongoDB VM POC not yet run
"npm": ProofMockOnly, // httptest emulator; real registry never hit
"gcp": ProofMockOnly, // httptest emulator (real RS256 JWT signing); no self-host
"twilio": ProofMockOnly, // httptest emulator; no self-host
"flyio": ProofMockOnly, // httptest GraphQL emulator; no self-host
"k8s": ProofMockOnly, // httptest apiserver emulator; real k3s POC not yet run
}
// ProofLevelOf returns the recorded proof level for a driver name. Unknown or