A Kubernetes operator that bridges Hardware Security Module (HSM) data storage with Kubernetes Secrets, providing true secret portability th
undo the runnable architecture
+76
-247
+46
-30
internal/modes/manager/manager.go
+46
-30
internal/modes/manager/manager.go
···
21
21
"flag"
22
22
"os"
23
23
"path/filepath"
24
+
"time"
24
25
25
26
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
26
27
// to ensure that exec-entrypoint and run can make use of them.
···
38
39
"sigs.k8s.io/controller-runtime/pkg/webhook"
39
40
40
41
hsmv1alpha1 "github.com/evanjarrett/hsm-secrets-operator/api/v1alpha1"
42
+
"github.com/evanjarrett/hsm-secrets-operator/internal/agent"
41
43
"github.com/evanjarrett/hsm-secrets-operator/internal/config"
42
44
"github.com/evanjarrett/hsm-secrets-operator/internal/controller"
43
45
)
···
242
244
return operatorNamespace, operatorName, nil
243
245
}
244
246
245
-
// setupBaseControllers sets up controllers that don't depend on the agent manager
246
-
func setupBaseControllers(mgr ctrl.Manager, cfg *managerConfig, serviceAccountName string) error {
247
+
// setupAllControllers sets up all controllers (base and agent-dependent)
248
+
func setupAllControllers(mgr ctrl.Manager, cfg *managerConfig, serviceAccountName string, agentManager *agent.Manager, operatorNamespace, operatorName string) error {
247
249
// Create image resolver
248
250
imageResolver := config.NewImageResolver(mgr.GetClient())
249
251
···
268
270
return err
269
271
}
270
272
273
+
// Set up HSMPool agent controller to deploy agents when pools are ready
274
+
if err := (&controller.HSMPoolAgentReconciler{
275
+
Client: mgr.GetClient(),
276
+
Scheme: mgr.GetScheme(),
277
+
AgentManager: agentManager,
278
+
ImageResolver: imageResolver,
279
+
DeviceAbsenceTimeout: 10 * time.Minute, // Default: cleanup agents after 10 minutes of device absence
280
+
ServiceAccountName: serviceAccountName,
281
+
}).SetupWithManager(mgr); err != nil {
282
+
setupLog.Error(err, "unable to create controller", "controller", "HSMPoolAgent")
283
+
return err
284
+
}
285
+
286
+
// Set up HSMSecret controller
287
+
if err := (&controller.HSMSecretReconciler{
288
+
Client: mgr.GetClient(),
289
+
Scheme: mgr.GetScheme(),
290
+
AgentManager: agentManager,
291
+
OperatorNamespace: operatorNamespace,
292
+
OperatorName: operatorName,
293
+
StartupTime: time.Now(),
294
+
}).SetupWithManager(mgr); err != nil {
295
+
setupLog.Error(err, "unable to create controller", "controller", "HSMSecret")
296
+
return err
297
+
}
298
+
299
+
setupLog.Info("All controllers set up successfully")
271
300
return nil
272
301
}
273
302
274
303
// startServices starts the API server, mirroring service, and manager
275
-
func startServices(mgr ctrl.Manager, agentManagerRunnable *AgentManagerRunnable, operatorNamespace string, cfg *managerConfig) error {
276
-
// Start API server if enabled (will wait for agent manager to be ready)
304
+
func startServices(mgr ctrl.Manager, agentManager *agent.Manager, operatorNamespace string, cfg *managerConfig) error {
305
+
// Start API server if enabled
277
306
if cfg.enableAPI {
278
307
// Create Kubernetes clientset for JWT authentication
279
308
k8sInterface, err := kubernetes.NewForConfig(mgr.GetConfig())
···
282
311
return err
283
312
}
284
313
285
-
// Create API server runnable that waits for agent manager
286
-
apiServerRunnable := NewAPIServerRunnable(mgr.GetClient(), agentManagerRunnable, operatorNamespace, k8sInterface, cfg.apiPort, ctrl.Log.WithName("api"))
314
+
// Create API server runnable
315
+
apiServerRunnable := NewAPIServerRunnable(mgr.GetClient(), agentManager, operatorNamespace, k8sInterface, cfg.apiPort, ctrl.Log.WithName("api"))
287
316
288
317
// Add API server as a Runnable to ensure it starts after the cache is ready
289
318
if err := mgr.Add(apiServerRunnable); err != nil {
290
319
setupLog.Error(err, "unable to add API server to manager")
291
320
return err
292
321
}
293
-
setupLog.Info("API server will start after agent manager is ready", "port", cfg.apiPort)
322
+
setupLog.Info("API server will start", "port", cfg.apiPort)
294
323
}
295
324
296
-
// Start device-scoped HSM mirroring in background (will wait for agent manager to be ready)
297
-
mirrorManagerRunnable := NewMirrorManagerRunnable(mgr.GetClient(), agentManagerRunnable, operatorNamespace, ctrl.Log.WithName("device-mirror"))
325
+
// Start device-scoped HSM mirroring in background
326
+
mirrorManagerRunnable := NewMirrorManagerRunnable(mgr.GetClient(), agentManager, operatorNamespace, ctrl.Log.WithName("device-mirror"))
298
327
if err := mgr.Add(mirrorManagerRunnable); err != nil {
299
328
setupLog.Error(err, "unable to add mirror manager to manager")
300
329
return err
301
330
}
302
-
setupLog.Info("Mirror manager will start after agent manager is ready")
331
+
setupLog.Info("Mirror manager will start")
303
332
304
333
setupLog.Info("starting manager")
305
334
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
···
334
363
}
335
364
setupLog.Info("Using service account", "serviceAccount", serviceAccountName)
336
365
337
-
// Create agent manager runnable that will create the agent manager after TLS is ready
338
-
agentManagerRunnable := NewAgentManagerRunnable(mgr.GetClient(), cfg.agentImage, operatorNamespace, serviceAccountName, setupLog)
366
+
// Create agent manager directly (no runnable needed)
367
+
setupLog.Info("Creating agent manager")
368
+
imageResolver := config.NewImageResolver(mgr.GetClient())
369
+
agentManager := agent.NewManager(mgr.GetClient(), operatorNamespace, cfg.agentImage, imageResolver)
339
370
340
-
// Add agent manager as a runnable to start after TLS is ready
341
-
setupLog.Info("Adding agent manager to manager")
342
-
if err := mgr.Add(agentManagerRunnable); err != nil {
343
-
setupLog.Error(err, "unable to add agent manager to manager")
344
-
return err
345
-
}
346
-
347
-
// Setup controllers that don't need the agent manager immediately
348
-
if err := setupBaseControllers(mgr, cfg, serviceAccountName); err != nil {
349
-
return err
350
-
}
351
-
352
-
// Create a runnable to setup agent-dependent controllers after agent manager is ready
353
-
agentControllerSetup := NewAgentControllerSetupRunnable(agentManagerRunnable, mgr, operatorNamespace, operatorName, serviceAccountName, setupLog)
354
-
setupLog.Info("Adding agent controller setup to manager")
355
-
if err := mgr.Add(agentControllerSetup); err != nil {
356
-
setupLog.Error(err, "unable to add agent controller setup to manager")
371
+
// Setup all controllers directly
372
+
if err := setupAllControllers(mgr, cfg, serviceAccountName, agentManager, operatorNamespace, operatorName); err != nil {
357
373
return err
358
374
}
359
375
···
382
398
return err
383
399
}
384
400
385
-
return startServices(mgr, agentManagerRunnable, operatorNamespace, cfg)
401
+
return startServices(mgr, agentManager, operatorNamespace, cfg)
386
402
}
+30
-217
internal/modes/manager/runnable.go
+30
-217
internal/modes/manager/runnable.go
···
19
19
import (
20
20
"context"
21
21
"fmt"
22
-
"sync"
23
22
"time"
24
23
25
24
"github.com/go-logr/logr"
26
25
"k8s.io/client-go/kubernetes"
27
26
"sigs.k8s.io/controller-runtime/pkg/client"
28
27
29
-
"sigs.k8s.io/controller-runtime/pkg/manager"
30
-
31
28
"github.com/evanjarrett/hsm-secrets-operator/internal/agent"
32
29
"github.com/evanjarrett/hsm-secrets-operator/internal/api"
33
-
"github.com/evanjarrett/hsm-secrets-operator/internal/config"
34
-
"github.com/evanjarrett/hsm-secrets-operator/internal/controller"
35
30
"github.com/evanjarrett/hsm-secrets-operator/internal/mirror"
36
31
)
37
32
38
-
// AgentManagerRunnable wraps an agent manager to create it after TLS is ready
39
-
type AgentManagerRunnable struct {
40
-
k8sClient client.Client
41
-
agentImage string
42
-
operatorNS string
43
-
serviceAccountName string
44
-
logger logr.Logger
45
-
agentManager *agent.Manager
46
-
mu sync.RWMutex
47
-
ready bool
48
-
readyCh chan struct{}
49
-
readyOnce sync.Once
50
-
}
51
-
52
-
// NewAgentManagerRunnable creates a new agent manager runnable
53
-
func NewAgentManagerRunnable(k8sClient client.Client, agentImage string, operatorNS, serviceAccountName string, logger logr.Logger) *AgentManagerRunnable {
54
-
return &AgentManagerRunnable{
55
-
k8sClient: k8sClient,
56
-
agentImage: agentImage,
57
-
operatorNS: operatorNS,
58
-
serviceAccountName: serviceAccountName,
59
-
logger: logger.WithName("agent-manager-runnable"),
60
-
readyCh: make(chan struct{}),
61
-
}
62
-
}
63
-
64
-
// Start creates the agent manager after TLS is ready - implements manager.Runnable
65
-
func (amr *AgentManagerRunnable) Start(ctx context.Context) error {
66
-
amr.logger.Info("Starting agent manager (after TLS is ready)")
67
-
68
-
// Create image resolver
69
-
imageResolver := config.NewImageResolver(amr.k8sClient)
70
-
71
-
// Create agent manager with TLS config
72
-
agentManager := agent.NewManager(amr.k8sClient, "", amr.agentImage, imageResolver)
73
-
74
-
// Store the agent manager and mark as ready
75
-
amr.mu.Lock()
76
-
amr.agentManager = agentManager
77
-
amr.ready = true
78
-
amr.mu.Unlock()
79
-
80
-
// Signal that agent manager is ready
81
-
amr.readyOnce.Do(func() {
82
-
close(amr.readyCh)
83
-
})
84
-
85
-
amr.logger.Info("Agent manager created successfully with TLS configuration")
86
-
87
-
// Wait for context cancellation (the agent manager itself doesn't need a shutdown sequence)
88
-
<-ctx.Done()
89
-
90
-
amr.logger.Info("Agent manager context cancelled")
91
-
return nil
92
-
}
93
-
94
-
// GetAgentManager returns the agent manager if ready, nil otherwise
95
-
func (amr *AgentManagerRunnable) GetAgentManager() *agent.Manager {
96
-
amr.mu.RLock()
97
-
defer amr.mu.RUnlock()
98
-
99
-
if !amr.ready {
100
-
return nil
101
-
}
102
-
return amr.agentManager
103
-
}
104
-
105
-
// WaitForReady waits for the agent manager to be ready
106
-
func (amr *AgentManagerRunnable) WaitForReady(ctx context.Context, timeout time.Duration) (*agent.Manager, error) {
107
-
// If already ready, return immediately
108
-
amr.mu.RLock()
109
-
if amr.ready {
110
-
agentManager := amr.agentManager
111
-
amr.mu.RUnlock()
112
-
return agentManager, nil
113
-
}
114
-
amr.mu.RUnlock()
115
-
116
-
// Wait for ready signal or timeout
117
-
select {
118
-
case <-amr.readyCh:
119
-
return amr.GetAgentManager(), nil
120
-
case <-time.After(timeout):
121
-
return nil, fmt.Errorf("timeout waiting for agent manager to be ready after %v", timeout)
122
-
case <-ctx.Done():
123
-
return nil, ctx.Err()
124
-
}
125
-
}
126
-
127
-
// IsReady returns true if the agent manager is ready
128
-
func (amr *AgentManagerRunnable) IsReady() bool {
129
-
amr.mu.RLock()
130
-
defer amr.mu.RUnlock()
131
-
return amr.ready
132
-
}
133
-
134
-
// APIServerRunnable starts the REST API server after agent manager is ready
33
+
// APIServerRunnable starts the REST API server
135
34
type APIServerRunnable struct {
136
-
k8sClient client.Client
137
-
agentManagerRunnable *AgentManagerRunnable
138
-
operatorNamespace string
139
-
k8sInterface *kubernetes.Clientset
140
-
apiPort int
141
-
logger logr.Logger
35
+
k8sClient client.Client
36
+
agentManager *agent.Manager
37
+
operatorNamespace string
38
+
k8sInterface *kubernetes.Clientset
39
+
apiPort int
40
+
logger logr.Logger
142
41
}
143
42
144
43
// NewAPIServerRunnable creates a new API server runnable
145
-
func NewAPIServerRunnable(k8sClient client.Client, agentManagerRunnable *AgentManagerRunnable, operatorNamespace string, k8sInterface *kubernetes.Clientset, apiPort int, logger logr.Logger) *APIServerRunnable {
44
+
func NewAPIServerRunnable(k8sClient client.Client, agentManager *agent.Manager, operatorNamespace string, k8sInterface *kubernetes.Clientset, apiPort int, logger logr.Logger) *APIServerRunnable {
146
45
return &APIServerRunnable{
147
-
k8sClient: k8sClient,
148
-
agentManagerRunnable: agentManagerRunnable,
149
-
operatorNamespace: operatorNamespace,
150
-
k8sInterface: k8sInterface,
151
-
apiPort: apiPort,
152
-
logger: logger.WithName("api-server-runnable"),
46
+
k8sClient: k8sClient,
47
+
agentManager: agentManager,
48
+
operatorNamespace: operatorNamespace,
49
+
k8sInterface: k8sInterface,
50
+
apiPort: apiPort,
51
+
logger: logger.WithName("api-server-runnable"),
153
52
}
154
53
}
155
54
156
-
// Start starts the API server after agent manager is ready - implements manager.Runnable
55
+
// Start starts the API server - implements manager.Runnable
157
56
func (asr *APIServerRunnable) Start(ctx context.Context) error {
158
-
asr.logger.Info("Waiting for agent manager to be ready before starting API server")
159
-
160
-
// Wait for agent manager to be ready
161
-
agentManager, err := asr.agentManagerRunnable.WaitForReady(ctx, 60*time.Second)
162
-
if err != nil {
163
-
return fmt.Errorf("timeout waiting for agent manager to be ready: %w", err)
164
-
}
165
-
166
-
asr.logger.Info("Agent manager is ready, starting API server", "port", asr.apiPort)
57
+
asr.logger.Info("Starting API server", "port", asr.apiPort)
167
58
168
59
// Start the API server
169
-
apiServer := api.NewServer(asr.k8sClient, agentManager, asr.operatorNamespace, asr.k8sInterface, asr.apiPort, asr.logger)
60
+
apiServer := api.NewServer(asr.k8sClient, asr.agentManager, asr.operatorNamespace, asr.k8sInterface, asr.apiPort, asr.logger)
170
61
return apiServer.Start(ctx)
171
62
}
172
63
173
-
// MirrorManagerRunnable starts the HSM mirroring service after agent manager is ready
64
+
// MirrorManagerRunnable starts the HSM mirroring service
174
65
type MirrorManagerRunnable struct {
175
-
k8sClient client.Client
176
-
agentManagerRunnable *AgentManagerRunnable
177
-
operatorNamespace string
178
-
logger logr.Logger
66
+
k8sClient client.Client
67
+
agentManager *agent.Manager
68
+
operatorNamespace string
69
+
logger logr.Logger
179
70
}
180
71
181
72
// NewMirrorManagerRunnable creates a new mirror manager runnable
182
-
func NewMirrorManagerRunnable(k8sClient client.Client, agentManagerRunnable *AgentManagerRunnable, operatorNamespace string, logger logr.Logger) *MirrorManagerRunnable {
73
+
func NewMirrorManagerRunnable(k8sClient client.Client, agentManager *agent.Manager, operatorNamespace string, logger logr.Logger) *MirrorManagerRunnable {
183
74
return &MirrorManagerRunnable{
184
-
k8sClient: k8sClient,
185
-
agentManagerRunnable: agentManagerRunnable,
186
-
operatorNamespace: operatorNamespace,
187
-
logger: logger.WithName("mirror-manager-runnable"),
75
+
k8sClient: k8sClient,
76
+
agentManager: agentManager,
77
+
operatorNamespace: operatorNamespace,
78
+
logger: logger.WithName("mirror-manager-runnable"),
188
79
}
189
80
}
190
81
191
-
// Start starts the mirroring service after agent manager is ready - implements manager.Runnable
82
+
// Start starts the mirroring service - implements manager.Runnable
192
83
func (mmr *MirrorManagerRunnable) Start(ctx context.Context) error {
193
-
mmr.logger.Info("Waiting for agent manager to be ready before starting HSM mirroring")
194
-
195
-
// Wait for agent manager to be ready
196
-
agentManager, err := mmr.agentManagerRunnable.WaitForReady(ctx, 60*time.Second)
197
-
if err != nil {
198
-
return fmt.Errorf("timeout waiting for agent manager to be ready: %w", err)
199
-
}
200
-
201
-
mmr.logger.Info("Agent manager is ready, starting HSM mirroring service")
84
+
mmr.logger.Info("Starting HSM mirroring service")
202
85
203
86
// Create mirror manager
204
-
mirrorManager := mirror.NewMirrorManager(mmr.k8sClient, agentManager, mmr.logger, mmr.operatorNamespace)
87
+
mirrorManager := mirror.NewMirrorManager(mmr.k8sClient, mmr.agentManager, mmr.logger, mmr.operatorNamespace)
205
88
206
89
// Start mirroring cycle
207
90
mirrorTicker := time.NewTicker(30 * time.Second) // Mirror every 30 seconds
···
251
134
}
252
135
}
253
136
}
254
-
255
-
// AgentControllerSetupRunnable sets up controllers that depend on the agent manager after it's ready
256
-
type AgentControllerSetupRunnable struct {
257
-
agentManagerRunnable *AgentManagerRunnable
258
-
mgr manager.Manager
259
-
operatorNamespace string
260
-
operatorName string
261
-
serviceAccountName string
262
-
logger logr.Logger
263
-
}
264
-
265
-
// NewAgentControllerSetupRunnable creates a new agent controller setup runnable
266
-
func NewAgentControllerSetupRunnable(agentManagerRunnable *AgentManagerRunnable, mgr manager.Manager, operatorNamespace, operatorName, serviceAccountName string, logger logr.Logger) *AgentControllerSetupRunnable {
267
-
return &AgentControllerSetupRunnable{
268
-
agentManagerRunnable: agentManagerRunnable,
269
-
mgr: mgr,
270
-
operatorNamespace: operatorNamespace,
271
-
operatorName: operatorName,
272
-
serviceAccountName: serviceAccountName,
273
-
logger: logger.WithName("agent-controller-setup"),
274
-
}
275
-
}
276
-
277
-
// Start sets up agent-dependent controllers after agent manager is ready - implements manager.Runnable
278
-
func (acsr *AgentControllerSetupRunnable) Start(ctx context.Context) error {
279
-
acsr.logger.Info("Waiting for agent manager to be ready before setting up controllers")
280
-
281
-
// Wait for agent manager to be ready
282
-
agentManager, err := acsr.agentManagerRunnable.WaitForReady(ctx, 60*time.Second)
283
-
if err != nil {
284
-
return fmt.Errorf("timeout waiting for agent manager to be ready: %w", err)
285
-
}
286
-
287
-
acsr.logger.Info("Agent manager is ready, setting up dependent controllers")
288
-
289
-
// Create image resolver
290
-
imageResolver := config.NewImageResolver(acsr.mgr.GetClient())
291
-
292
-
// Set up HSMPool agent controller to deploy agents when pools are ready
293
-
if err := (&controller.HSMPoolAgentReconciler{
294
-
Client: acsr.mgr.GetClient(),
295
-
Scheme: acsr.mgr.GetScheme(),
296
-
AgentManager: agentManager,
297
-
ImageResolver: imageResolver,
298
-
DeviceAbsenceTimeout: 10 * time.Minute, // Default: cleanup agents after 10 minutes of device absence
299
-
ServiceAccountName: acsr.serviceAccountName,
300
-
}).SetupWithManager(acsr.mgr); err != nil {
301
-
return fmt.Errorf("unable to create controller HSMPoolAgent: %w", err)
302
-
}
303
-
304
-
// Set up HSMSecret controller
305
-
if err := (&controller.HSMSecretReconciler{
306
-
Client: acsr.mgr.GetClient(),
307
-
Scheme: acsr.mgr.GetScheme(),
308
-
AgentManager: agentManager,
309
-
OperatorNamespace: acsr.operatorNamespace,
310
-
OperatorName: acsr.operatorName,
311
-
StartupTime: time.Now(),
312
-
}).SetupWithManager(acsr.mgr); err != nil {
313
-
return fmt.Errorf("unable to create controller HSMSecret: %w", err)
314
-
}
315
-
316
-
acsr.logger.Info("Agent-dependent controllers set up successfully")
317
-
318
-
// Wait for context cancellation
319
-
<-ctx.Done()
320
-
321
-
acsr.logger.Info("Agent controller setup context cancelled")
322
-
return nil
323
-
}