A Kubernetes operator that bridges Hardware Security Module (HSM) data storage with Kubernetes Secrets, providing true secret portability th
1/*
2Copyright 2025.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17package controller
18
19import (
20 "context"
21 "fmt"
22
23 . "github.com/onsi/ginkgo/v2"
24 . "github.com/onsi/gomega"
25 appsv1 "k8s.io/api/apps/v1"
26 corev1 "k8s.io/api/core/v1"
27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28 "k8s.io/apimachinery/pkg/types"
29 "k8s.io/apimachinery/pkg/util/intstr"
30 ctrl "sigs.k8s.io/controller-runtime"
31
32 hsmv1alpha1 "github.com/evanjarrett/hsm-secrets-operator/api/v1alpha1"
33)
34
35var _ = Describe("HSMPoolReconciler with Manager-Owned DaemonSets", func() {
36 Context("When reconciling an HSMPool with DaemonSet pods", func() {
37 var (
38 hsmDeviceName string
39 hsmPoolName string
40 hsmPoolNamespace = "default"
41 )
42
43 ctx := context.Background()
44
45 var hsmDevice *hsmv1alpha1.HSMDevice
46 var hsmPool *hsmv1alpha1.HSMPool
47 var daemonSet *appsv1.DaemonSet
48
49 BeforeEach(func() {
50 // Generate unique resource names for each test
51 hsmDeviceName = fmt.Sprintf("test-pool-device-%d", GinkgoRandomSeed())
52 hsmPoolName = fmt.Sprintf("test-pool-%d", GinkgoRandomSeed())
53
54 // Create HSMDevice
55 hsmDevice = &hsmv1alpha1.HSMDevice{
56 ObjectMeta: metav1.ObjectMeta{
57 Name: hsmDeviceName,
58 Namespace: hsmPoolNamespace,
59 },
60 Spec: hsmv1alpha1.HSMDeviceSpec{
61 DeviceType: "PicoHSM",
62 Discovery: &hsmv1alpha1.DiscoverySpec{
63 USB: &hsmv1alpha1.USBDeviceSpec{
64 VendorID: "20a0",
65 ProductID: "4230",
66 },
67 },
68 },
69 }
70 Expect(k8sClient.Create(ctx, hsmDevice)).To(Succeed())
71
72 // Create DaemonSet owned by HSMDevice (simulating DiscoveryDaemonSetReconciler)
73 daemonSet = &appsv1.DaemonSet{
74 ObjectMeta: metav1.ObjectMeta{
75 Name: fmt.Sprintf("%s-discovery", hsmDeviceName),
76 Namespace: hsmPoolNamespace,
77 Labels: map[string]string{
78 "app.kubernetes.io/name": "hsm-secrets-operator",
79 "app.kubernetes.io/component": "discovery",
80 "hsm.j5t.io/device": hsmDeviceName,
81 },
82 OwnerReferences: []metav1.OwnerReference{
83 {
84 APIVersion: "hsm.j5t.io/v1alpha1",
85 Kind: "HSMDevice",
86 Name: hsmDeviceName,
87 UID: hsmDevice.UID,
88 Controller: &[]bool{true}[0],
89 },
90 },
91 },
92 Spec: appsv1.DaemonSetSpec{
93 Selector: &metav1.LabelSelector{
94 MatchLabels: map[string]string{
95 "app.kubernetes.io/name": "hsm-secrets-operator",
96 "app.kubernetes.io/component": "discovery",
97 "hsm.j5t.io/device": hsmDeviceName,
98 },
99 },
100 Template: corev1.PodTemplateSpec{
101 ObjectMeta: metav1.ObjectMeta{
102 Labels: map[string]string{
103 "app.kubernetes.io/name": "hsm-secrets-operator",
104 "app.kubernetes.io/component": "discovery",
105 "hsm.j5t.io/device": hsmDeviceName,
106 },
107 },
108 Spec: corev1.PodSpec{
109 Containers: []corev1.Container{
110 {
111 Name: "discovery",
112 Image: "test-discovery:latest",
113 },
114 },
115 },
116 },
117 UpdateStrategy: appsv1.DaemonSetUpdateStrategy{
118 Type: appsv1.RollingUpdateDaemonSetStrategyType,
119 RollingUpdate: &appsv1.RollingUpdateDaemonSet{
120 MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1},
121 },
122 },
123 },
124 Status: appsv1.DaemonSetStatus{
125 DesiredNumberScheduled: 2,
126 NumberReady: 1,
127 },
128 }
129 Expect(k8sClient.Create(ctx, daemonSet)).To(Succeed())
130
131 // Update DaemonSet status separately (status is not created with the resource)
132 daemonSet.Status = appsv1.DaemonSetStatus{
133 DesiredNumberScheduled: 2,
134 NumberReady: 1,
135 }
136 Expect(k8sClient.Status().Update(ctx, daemonSet)).To(Succeed())
137
138 // Create HSMPool referencing the HSMDevice
139 hsmPool = &hsmv1alpha1.HSMPool{
140 ObjectMeta: metav1.ObjectMeta{
141 Name: hsmPoolName,
142 Namespace: hsmPoolNamespace,
143 OwnerReferences: []metav1.OwnerReference{
144 {
145 APIVersion: "hsm.j5t.io/v1alpha1",
146 Kind: "HSMDevice",
147 Name: hsmDevice.Name,
148 UID: hsmDevice.UID,
149 },
150 },
151 },
152 Spec: hsmv1alpha1.HSMPoolSpec{},
153 }
154 Expect(k8sClient.Create(ctx, hsmPool)).To(Succeed())
155 })
156
157 AfterEach(func() {
158 // Clean up resources
159 if hsmPool != nil {
160 _ = k8sClient.Delete(ctx, hsmPool)
161 }
162 if daemonSet != nil {
163 _ = k8sClient.Delete(ctx, daemonSet)
164 }
165 if hsmDevice != nil {
166 _ = k8sClient.Delete(ctx, hsmDevice)
167 }
168 })
169
170 It("Should collect pod reports from DaemonSet pods directly", func() {
171 By("Creating some discovery pods for the DaemonSet")
172 runningPod := &corev1.Pod{
173 ObjectMeta: metav1.ObjectMeta{
174 Name: fmt.Sprintf("%s-pod-1", hsmDeviceName),
175 Namespace: hsmPoolNamespace,
176 Labels: map[string]string{
177 "app.kubernetes.io/name": "hsm-secrets-operator",
178 "app.kubernetes.io/component": "discovery",
179 "hsm.j5t.io/device": hsmDeviceName,
180 },
181 },
182 Spec: corev1.PodSpec{
183 NodeName: "node-1",
184 Containers: []corev1.Container{
185 {
186 Name: "discovery",
187 Image: "test-discovery:latest",
188 },
189 },
190 },
191 Status: corev1.PodStatus{
192 Phase: corev1.PodRunning,
193 },
194 }
195 Expect(k8sClient.Create(ctx, runningPod)).To(Succeed())
196
197 // Update pod status separately (status is not created with the resource)
198 runningPod.Status = corev1.PodStatus{
199 Phase: corev1.PodRunning,
200 }
201 Expect(k8sClient.Status().Update(ctx, runningPod)).To(Succeed())
202
203 pendingPod := &corev1.Pod{
204 ObjectMeta: metav1.ObjectMeta{
205 Name: fmt.Sprintf("%s-pod-2", hsmDeviceName),
206 Namespace: hsmPoolNamespace,
207 Labels: map[string]string{
208 "app.kubernetes.io/name": "hsm-secrets-operator",
209 "app.kubernetes.io/component": "discovery",
210 "hsm.j5t.io/device": hsmDeviceName,
211 },
212 },
213 Spec: corev1.PodSpec{
214 NodeName: "node-2",
215 Containers: []corev1.Container{
216 {
217 Name: "discovery",
218 Image: "test-discovery:latest",
219 },
220 },
221 },
222 Status: corev1.PodStatus{
223 Phase: corev1.PodPending,
224 },
225 }
226 Expect(k8sClient.Create(ctx, pendingPod)).To(Succeed())
227
228 // Update pod status separately (status is not created with the resource)
229 pendingPod.Status = corev1.PodStatus{
230 Phase: corev1.PodPending,
231 }
232 Expect(k8sClient.Status().Update(ctx, pendingPod)).To(Succeed())
233
234 By("Reconciling the HSMPool")
235 reconciler := &HSMPoolReconciler{
236 Client: k8sClient,
237 Scheme: k8sClient.Scheme(),
238 }
239
240 _, err := reconciler.Reconcile(ctx, ctrl.Request{
241 NamespacedName: types.NamespacedName{
242 Name: hsmPoolName,
243 Namespace: hsmPoolNamespace,
244 },
245 })
246 Expect(err).NotTo(HaveOccurred())
247
248 By("Checking that HSMPool status was updated with pod information")
249 pool := &hsmv1alpha1.HSMPool{}
250 Eventually(func() bool {
251 _ = k8sClient.Get(ctx, types.NamespacedName{
252 Name: hsmPoolName,
253 Namespace: hsmPoolNamespace,
254 }, pool)
255 return len(pool.Status.ReportingPods) > 0
256 }).Should(BeTrue())
257
258 // Verify reporting pods were collected from DaemonSet
259 Expect(pool.Status.ReportingPods).To(HaveLen(2))
260
261 var runningReport, pendingReport *hsmv1alpha1.PodReport
262 for i := range pool.Status.ReportingPods {
263 report := &pool.Status.ReportingPods[i]
264 switch report.PodName {
265 case runningPod.Name:
266 runningReport = report
267 case pendingPod.Name:
268 pendingReport = report
269 }
270 }
271
272 Expect(runningReport).NotTo(BeNil())
273 Expect(runningReport.NodeName).To(Equal("node-1"))
274 Expect(runningReport.DiscoveryStatus).To(Equal("completed"))
275 Expect(runningReport.DevicesFound).To(Equal(int32(0))) // Running pod without annotation reports 0 devices
276 Expect(runningReport.Fresh).To(BeTrue())
277
278 Expect(pendingReport).NotTo(BeNil())
279 Expect(pendingReport.NodeName).To(Equal("node-2"))
280 Expect(pendingReport.DiscoveryStatus).To(Equal("pending"))
281 Expect(pendingReport.DevicesFound).To(Equal(int32(0))) // Pending pod reports 0 devices
282 Expect(pendingReport.Fresh).To(BeFalse()) // Pending pods are not considered fresh
283
284 // Verify expected pods count from DaemonSet status
285 Expect(pool.Status.ExpectedPods).To(Equal(int32(2)))
286 })
287
288 It("Should handle missing DaemonSet gracefully", func() {
289 By("Creating HSMPool referencing non-existent HSMDevice")
290 missingPoolName := fmt.Sprintf("missing-pool-%d", GinkgoRandomSeed())
291 missingPool := &hsmv1alpha1.HSMPool{
292 ObjectMeta: metav1.ObjectMeta{
293 Name: missingPoolName,
294 Namespace: hsmPoolNamespace,
295 OwnerReferences: []metav1.OwnerReference{
296 {
297 APIVersion: "hsm.j5t.io/v1alpha1",
298 Kind: "HSMDevice",
299 Name: "non-existent-device",
300 UID: "fake-uid-456",
301 },
302 },
303 },
304 Spec: hsmv1alpha1.HSMPoolSpec{},
305 }
306 Expect(k8sClient.Create(ctx, missingPool)).To(Succeed())
307
308 By("Reconciling the HSMPool with missing HSMDevice")
309 reconciler := &HSMPoolReconciler{
310 Client: k8sClient,
311 Scheme: k8sClient.Scheme(),
312 }
313
314 _, err := reconciler.Reconcile(ctx, ctrl.Request{
315 NamespacedName: types.NamespacedName{
316 Name: missingPoolName,
317 Namespace: hsmPoolNamespace,
318 },
319 })
320 Expect(err).NotTo(HaveOccurred())
321
322 By("Checking that HSMPool status indicates error")
323 pool := &hsmv1alpha1.HSMPool{}
324 Eventually(func() hsmv1alpha1.HSMPoolPhase {
325 _ = k8sClient.Get(ctx, types.NamespacedName{
326 Name: missingPoolName,
327 Namespace: hsmPoolNamespace,
328 }, pool)
329 return pool.Status.Phase
330 }).Should(Equal(hsmv1alpha1.HSMPoolPhaseError))
331 })
332
333 It("Should read device count from pod annotations when available", func() {
334 By("Creating a discovery pod with annotation-based device report")
335 podWithAnnotation := &corev1.Pod{
336 ObjectMeta: metav1.ObjectMeta{
337 Name: fmt.Sprintf("%s-annotated-pod", hsmDeviceName),
338 Namespace: hsmPoolNamespace,
339 Labels: map[string]string{
340 "app.kubernetes.io/name": "hsm-secrets-operator",
341 "app.kubernetes.io/component": "discovery",
342 "hsm.j5t.io/device": hsmDeviceName,
343 },
344 Annotations: map[string]string{
345 "hsm.j5t.io/device-report": `{
346 "hsmDeviceName": "` + hsmDeviceName + `",
347 "reportingNode": "node-1",
348 "discoveredDevices": [
349 {
350 "devicePath": "/dev/bus/usb/001/015",
351 "serialNumber": "DC6A33145E23A42A",
352 "nodeName": "node-1",
353 "lastSeen": "2025-08-19T10:00:00Z",
354 "available": true
355 },
356 {
357 "devicePath": "/dev/bus/usb/001/016",
358 "serialNumber": "DC6A33145E23A42B",
359 "nodeName": "node-1",
360 "lastSeen": "2025-08-19T10:00:00Z",
361 "available": true
362 }
363 ],
364 "lastReportTime": "2025-08-19T10:00:00Z",
365 "discoveryStatus": "completed"
366 }`,
367 },
368 },
369 Spec: corev1.PodSpec{
370 NodeName: "node-1",
371 Containers: []corev1.Container{
372 {
373 Name: "discovery",
374 Image: "test-discovery:latest",
375 },
376 },
377 },
378 Status: corev1.PodStatus{
379 Phase: corev1.PodRunning,
380 },
381 }
382 Expect(k8sClient.Create(ctx, podWithAnnotation)).To(Succeed())
383
384 // Update pod status separately (status is not created with the resource)
385 podWithAnnotation.Status = corev1.PodStatus{
386 Phase: corev1.PodRunning,
387 }
388 Expect(k8sClient.Status().Update(ctx, podWithAnnotation)).To(Succeed())
389
390 By("Reconciling the HSMPool")
391 reconciler := &HSMPoolReconciler{
392 Client: k8sClient,
393 Scheme: k8sClient.Scheme(),
394 }
395
396 _, err := reconciler.Reconcile(ctx, ctrl.Request{
397 NamespacedName: types.NamespacedName{
398 Name: hsmPoolName,
399 Namespace: hsmPoolNamespace,
400 },
401 })
402 Expect(err).NotTo(HaveOccurred())
403
404 By("Checking that HSMPool reads device count from annotation")
405 pool := &hsmv1alpha1.HSMPool{}
406 Eventually(func() bool {
407 _ = k8sClient.Get(ctx, types.NamespacedName{
408 Name: hsmPoolName,
409 Namespace: hsmPoolNamespace,
410 }, pool)
411
412 // Find the pod with annotation
413 for _, report := range pool.Status.ReportingPods {
414 if report.PodName == podWithAnnotation.Name {
415 return report.DevicesFound == 2 // Should read 2 devices from annotation
416 }
417 }
418 return false
419 }).Should(BeTrue())
420
421 // Verify the specific report details
422 var annotatedReport *hsmv1alpha1.PodReport
423 for i := range pool.Status.ReportingPods {
424 report := &pool.Status.ReportingPods[i]
425 if report.PodName == podWithAnnotation.Name {
426 annotatedReport = report
427 break
428 }
429 }
430
431 Expect(annotatedReport).NotTo(BeNil())
432 Expect(annotatedReport.DevicesFound).To(Equal(int32(2))) // 2 devices from annotation
433 Expect(annotatedReport.DiscoveryStatus).To(Equal("completed")) // Status from annotation
434 Expect(annotatedReport.NodeName).To(Equal("node-1"))
435
436 // Verify that TotalDevices and AvailableDevices are correctly aggregated
437 Expect(pool.Status.TotalDevices).To(Equal(int32(2))) // Should aggregate 2 devices total
438 Expect(pool.Status.AvailableDevices).To(Equal(int32(2))) // Both devices are available
439 })
440 })
441})