A container registry that uses the AT Protocol for manifest storage and S3 for blob storage.
atcr.io
docker
container
atproto
go
1package main
2
3import (
4 "bufio"
5 "context"
6 crypto_rand "crypto/rand"
7 "crypto/sha256"
8 "encoding/base64"
9 "encoding/hex"
10 "fmt"
11 "os"
12 "path/filepath"
13 "strings"
14 "time"
15
16 "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud"
17 "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/request"
18 "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/service"
19 "github.com/spf13/cobra"
20)
21
22var provisionCmd = &cobra.Command{
23 Use: "provision",
24 Short: "Create all infrastructure (servers, network, LB, firewall)",
25 RunE: func(cmd *cobra.Command, args []string) error {
26 token, _ := cmd.Root().PersistentFlags().GetString("token")
27 zone, _ := cmd.Flags().GetString("zone")
28 plan, _ := cmd.Flags().GetString("plan")
29 sshKey, _ := cmd.Flags().GetString("ssh-key")
30 s3Secret, _ := cmd.Flags().GetString("s3-secret")
31 withScanner, _ := cmd.Flags().GetBool("with-scanner")
32 return cmdProvision(token, zone, plan, sshKey, s3Secret, withScanner)
33 },
34}
35
36func init() {
37 provisionCmd.Flags().String("zone", "", "UpCloud zone (interactive picker if omitted)")
38 provisionCmd.Flags().String("plan", "", "Server plan (interactive picker if omitted)")
39 provisionCmd.Flags().String("ssh-key", "", "Path to SSH public key file (required)")
40 provisionCmd.Flags().String("s3-secret", "", "S3 secret access key (for existing object storage)")
41 provisionCmd.Flags().Bool("with-scanner", false, "Deploy vulnerability scanner alongside hold")
42 _ = provisionCmd.MarkFlagRequired("ssh-key")
43 rootCmd.AddCommand(provisionCmd)
44}
45
46func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner bool) error {
47 cfg, err := loadConfig(zone, plan, sshKeyPath, s3Secret)
48 if err != nil {
49 return err
50 }
51
52 naming := cfg.Naming()
53
54 svc, err := newService(token)
55 if err != nil {
56 return err
57 }
58
59 ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
60 defer cancel()
61
62 // Load existing state or start fresh
63 state, err := loadState()
64 if err != nil {
65 state = &InfraState{}
66 }
67
68 // Use zone from state if not provided via flags
69 if cfg.Zone == "" && state.Zone != "" {
70 cfg.Zone = state.Zone
71 }
72
73 // Only need interactive picker if we still need to create resources
74 needsServers := state.Appview.UUID == "" || state.Hold.UUID == ""
75 if cfg.Zone == "" || (needsServers && cfg.Plan == "") {
76 if err := resolveInteractive(ctx, svc, cfg); err != nil {
77 return err
78 }
79 }
80
81 if state.Zone == "" {
82 state.Zone = cfg.Zone
83 }
84 state.ClientName = cfg.ClientName
85 state.RepoBranch = cfg.RepoBranch
86
87 // Scanner setup
88 if withScanner {
89 state.ScannerEnabled = true
90 if state.ScannerSecret == "" {
91 secret, err := generateScannerSecret()
92 if err != nil {
93 return fmt.Errorf("generate scanner secret: %w", err)
94 }
95 state.ScannerSecret = secret
96 fmt.Printf("Generated scanner shared secret\n")
97 }
98 _ = saveState(state)
99 }
100
101 fmt.Printf("Provisioning %s infrastructure in zone %s...\n", naming.DisplayName(), cfg.Zone)
102 if needsServers {
103 fmt.Printf("Server plan: %s\n", cfg.Plan)
104 }
105 fmt.Println()
106
107 // S3 secret key — from flag for existing storage, from API for new
108 s3SecretKey := cfg.S3SecretKey
109
110 // 1. Object storage
111 if state.ObjectStorage.UUID != "" {
112 fmt.Printf("Object storage: %s (exists)\n", state.ObjectStorage.UUID)
113 // Refresh discoverable fields if missing (e.g. pre-seeded UUID only)
114 if state.ObjectStorage.Endpoint == "" || state.ObjectStorage.Bucket == "" {
115 fmt.Println(" Discovering endpoint, bucket, access key...")
116 discovered, err := lookupObjectStorage(ctx, svc, state.ObjectStorage.UUID)
117 if err != nil {
118 return err
119 }
120 state.ObjectStorage.Endpoint = discovered.Endpoint
121 state.ObjectStorage.Region = discovered.Region
122 if discovered.Bucket != "" {
123 state.ObjectStorage.Bucket = discovered.Bucket
124 }
125 if discovered.AccessKeyID != "" {
126 state.ObjectStorage.AccessKeyID = discovered.AccessKeyID
127 }
128 _ = saveState(state)
129 }
130 } else {
131 fmt.Println("Creating object storage...")
132 objState, secretKey, err := provisionObjectStorage(ctx, svc, cfg.Zone, naming.S3Name())
133 if err != nil {
134 return fmt.Errorf("object storage: %w", err)
135 }
136 state.ObjectStorage = objState
137 s3SecretKey = secretKey
138 _ = saveState(state)
139 fmt.Printf(" S3 Secret Key: %s\n", secretKey)
140 }
141
142 fmt.Printf(" Endpoint: %s\n", state.ObjectStorage.Endpoint)
143 fmt.Printf(" Region: %s\n", state.ObjectStorage.Region)
144 fmt.Printf(" Bucket: %s\n", state.ObjectStorage.Bucket)
145 fmt.Printf(" Access Key: %s\n\n", state.ObjectStorage.AccessKeyID)
146
147 // Hold domain is zone-based (e.g. us-chi1.cove.seamark.dev)
148 holdDomain := cfg.Zone + ".cove." + cfg.BaseDomain
149
150 // Build config template values
151 vals := &ConfigValues{
152 S3Endpoint: state.ObjectStorage.Endpoint,
153 S3Region: state.ObjectStorage.Region,
154 S3Bucket: state.ObjectStorage.Bucket,
155 S3AccessKey: state.ObjectStorage.AccessKeyID,
156 S3SecretKey: s3SecretKey,
157 Zone: cfg.Zone,
158 HoldDomain: holdDomain,
159 HoldDid: "did:web:" + holdDomain,
160 BasePath: naming.BasePath(),
161 ScannerSecret: state.ScannerSecret,
162 }
163
164 // 2. Private network
165 if state.Network.UUID != "" {
166 fmt.Printf("Network: %s (exists)\n", state.Network.UUID)
167 } else {
168 fmt.Println("Creating private network...")
169 network, err := svc.CreateNetwork(ctx, &request.CreateNetworkRequest{
170 Name: naming.NetworkName(),
171 Zone: cfg.Zone,
172 IPNetworks: upcloud.IPNetworkSlice{
173 {
174 Address: privateNetworkCIDR,
175 DHCP: upcloud.True,
176 DHCPDefaultRoute: upcloud.False,
177 DHCPDns: []string{"8.8.8.8", "1.1.1.1"},
178 Family: upcloud.IPAddressFamilyIPv4,
179 Gateway: "",
180 },
181 },
182 })
183 if err != nil {
184 return fmt.Errorf("create network: %w", err)
185 }
186 state.Network = StateRef{UUID: network.UUID}
187 _ = saveState(state)
188 fmt.Printf(" Network: %s (%s)\n", network.UUID, privateNetworkCIDR)
189 }
190
191 // Find Debian template (needed for server creation)
192 templateUUID, err := findDebianTemplate(ctx, svc)
193 if err != nil {
194 return err
195 }
196
197 // 3. Appview server
198 appviewCreated := false
199 if state.Appview.UUID != "" {
200 fmt.Printf("Appview: %s (exists)\n", state.Appview.UUID)
201 appviewScript, err := generateAppviewCloudInit(cfg, vals)
202 if err != nil {
203 return err
204 }
205 if err := syncCloudInit("appview", state.Appview.PublicIP, appviewScript); err != nil {
206 return err
207 }
208 appviewConfigYAML, err := renderConfig(appviewConfigTmpl, vals)
209 if err != nil {
210 return fmt.Errorf("render appview config: %w", err)
211 }
212 if err := syncConfigKeys("appview", state.Appview.PublicIP, naming.AppviewConfigPath(), appviewConfigYAML); err != nil {
213 return fmt.Errorf("appview config sync: %w", err)
214 }
215 } else {
216 fmt.Println("Creating appview server...")
217 appviewUserData, err := generateAppviewCloudInit(cfg, vals)
218 if err != nil {
219 return err
220 }
221 appview, err := createServer(ctx, svc, cfg, templateUUID, state.Network.UUID, naming.Appview(), appviewUserData)
222 if err != nil {
223 return fmt.Errorf("create appview: %w", err)
224 }
225 state.Appview = *appview
226 _ = saveState(state)
227 appviewCreated = true
228 fmt.Printf(" Appview: %s (public: %s, private: %s)\n", appview.UUID, appview.PublicIP, appview.PrivateIP)
229 }
230
231 // 4. Hold server
232 holdCreated := false
233 if state.Hold.UUID != "" {
234 fmt.Printf("Hold: %s (exists)\n", state.Hold.UUID)
235 holdScript, err := generateHoldCloudInit(cfg, vals, state.ScannerEnabled)
236 if err != nil {
237 return err
238 }
239 if err := syncCloudInit("hold", state.Hold.PublicIP, holdScript); err != nil {
240 return err
241 }
242 holdConfigYAML, err := renderConfig(holdConfigTmpl, vals)
243 if err != nil {
244 return fmt.Errorf("render hold config: %w", err)
245 }
246 if err := syncConfigKeys("hold", state.Hold.PublicIP, naming.HoldConfigPath(), holdConfigYAML); err != nil {
247 return fmt.Errorf("hold config sync: %w", err)
248 }
249 if state.ScannerEnabled {
250 scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
251 if err != nil {
252 return fmt.Errorf("render scanner config: %w", err)
253 }
254 if err := syncConfigKeys("scanner", state.Hold.PublicIP, naming.ScannerConfigPath(), scannerConfigYAML); err != nil {
255 return fmt.Errorf("scanner config sync: %w", err)
256 }
257 }
258 } else {
259 fmt.Println("Creating hold server...")
260 holdUserData, err := generateHoldCloudInit(cfg, vals, state.ScannerEnabled)
261 if err != nil {
262 return err
263 }
264 hold, err := createServer(ctx, svc, cfg, templateUUID, state.Network.UUID, naming.Hold(), holdUserData)
265 if err != nil {
266 return fmt.Errorf("create hold: %w", err)
267 }
268 state.Hold = *hold
269 _ = saveState(state)
270 holdCreated = true
271 fmt.Printf(" Hold: %s (public: %s, private: %s)\n", hold.UUID, hold.PublicIP, hold.PrivateIP)
272 }
273
274 // 5. Firewall rules (idempotent — replaces all rules)
275 fmt.Println("Configuring firewall rules...")
276 for _, s := range []struct {
277 name string
278 uuid string
279 }{
280 {"appview", state.Appview.UUID},
281 {"hold", state.Hold.UUID},
282 } {
283 if err := createFirewallRules(ctx, svc, s.uuid, privateNetworkCIDR); err != nil {
284 return fmt.Errorf("firewall %s: %w", s.name, err)
285 }
286 }
287
288 // 6. Load balancer
289 if state.LB.UUID != "" {
290 fmt.Printf("Load balancer: %s (exists)\n", state.LB.UUID)
291 } else {
292 fmt.Println("Creating load balancer (Essentials tier)...")
293 lb, err := createLoadBalancer(ctx, svc, cfg, naming, state.Network.UUID, state.Appview.PrivateIP, state.Hold.PrivateIP, holdDomain)
294 if err != nil {
295 return fmt.Errorf("create LB: %w", err)
296 }
297 state.LB = StateRef{UUID: lb.UUID}
298 _ = saveState(state)
299 }
300
301 // Always reconcile forwarded headers rule (handles existing LBs)
302 if err := ensureLBForwardedHeaders(ctx, svc, state.LB.UUID); err != nil {
303 return fmt.Errorf("LB forwarded headers: %w", err)
304 }
305
306 // Ensure route-hold rule includes forwarded headers action
307 if err := ensureLBHoldForwardedHeaders(ctx, svc, state.LB.UUID, holdDomain); err != nil {
308 return fmt.Errorf("LB hold forwarded headers: %w", err)
309 }
310
311 // Always reconcile scanner block rule
312 if err := ensureLBScannerBlock(ctx, svc, state.LB.UUID); err != nil {
313 return fmt.Errorf("LB scanner block: %w", err)
314 }
315
316 // Always reconcile TLS certs (handles partial failures and re-runs)
317 tlsDomains := []string{cfg.BaseDomain}
318 tlsDomains = append(tlsDomains, cfg.RegistryDomains...)
319 tlsDomains = append(tlsDomains, holdDomain)
320 if err := ensureLBCertificates(ctx, svc, state.LB.UUID, tlsDomains); err != nil {
321 return fmt.Errorf("LB certificates: %w", err)
322 }
323
324 // Fetch LB DNS name for output
325 lbDNS := ""
326 if state.LB.UUID != "" {
327 lb, err := svc.GetLoadBalancer(ctx, &request.GetLoadBalancerRequest{UUID: state.LB.UUID})
328 if err == nil {
329 for _, n := range lb.Networks {
330 if n.Type == upcloud.LoadBalancerNetworkTypePublic {
331 lbDNS = n.DNSName
332 }
333 }
334 }
335 }
336
337 // 7. Build locally and deploy binaries to new servers
338 if appviewCreated || holdCreated {
339 rootDir := projectRoot()
340
341 if err := runGenerate(rootDir); err != nil {
342 return fmt.Errorf("go generate: %w", err)
343 }
344
345 fmt.Println("\nBuilding locally (GOOS=linux GOARCH=amd64)...")
346 if appviewCreated {
347 outputPath := filepath.Join(rootDir, "bin", "atcr-appview")
348 if err := buildLocal(rootDir, outputPath, "./cmd/appview"); err != nil {
349 return fmt.Errorf("build appview: %w", err)
350 }
351 }
352 if holdCreated {
353 outputPath := filepath.Join(rootDir, "bin", "atcr-hold")
354 if err := buildLocal(rootDir, outputPath, "./cmd/hold"); err != nil {
355 return fmt.Errorf("build hold: %w", err)
356 }
357 if state.ScannerEnabled {
358 outputPath := filepath.Join(rootDir, "bin", "atcr-scanner")
359 if err := buildLocal(filepath.Join(rootDir, "scanner"), outputPath, "./cmd/scanner"); err != nil {
360 return fmt.Errorf("build scanner: %w", err)
361 }
362 }
363 }
364
365 fmt.Println("\nWaiting for cloud-init to complete on new servers...")
366 if appviewCreated {
367 if err := waitForSetup(state.Appview.PublicIP, "appview"); err != nil {
368 return err
369 }
370 }
371 if holdCreated {
372 if err := waitForSetup(state.Hold.PublicIP, "hold"); err != nil {
373 return err
374 }
375 }
376
377 fmt.Println("\nDeploying binaries...")
378 if appviewCreated {
379 localPath := filepath.Join(rootDir, "bin", "atcr-appview")
380 remotePath := naming.InstallDir() + "/bin/" + naming.Appview()
381 if err := scpFile(localPath, state.Appview.PublicIP, remotePath); err != nil {
382 return fmt.Errorf("upload appview: %w", err)
383 }
384 }
385 if holdCreated {
386 localPath := filepath.Join(rootDir, "bin", "atcr-hold")
387 remotePath := naming.InstallDir() + "/bin/" + naming.Hold()
388 if err := scpFile(localPath, state.Hold.PublicIP, remotePath); err != nil {
389 return fmt.Errorf("upload hold: %w", err)
390 }
391 if state.ScannerEnabled {
392 scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
393 scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
394 if err := scpFile(scannerLocal, state.Hold.PublicIP, scannerRemote); err != nil {
395 return fmt.Errorf("upload scanner: %w", err)
396 }
397 }
398 }
399 }
400
401 fmt.Println("\n=== Provisioning Complete ===")
402 fmt.Println()
403 fmt.Println("DNS records needed:")
404 if lbDNS != "" {
405 fmt.Printf(" CNAME %-24s → %s\n", cfg.BaseDomain, lbDNS)
406 for _, rd := range cfg.RegistryDomains {
407 fmt.Printf(" CNAME %-24s → %s\n", rd, lbDNS)
408 }
409 fmt.Printf(" CNAME %-24s → %s\n", holdDomain, lbDNS)
410 } else {
411 fmt.Println(" (LB DNS name not yet available — check 'status' in a few minutes)")
412 }
413 fmt.Println()
414 fmt.Println("SSH access:")
415 fmt.Printf(" ssh root@%s # appview\n", state.Appview.PublicIP)
416 fmt.Printf(" ssh root@%s # hold\n", state.Hold.PublicIP)
417 fmt.Println()
418 fmt.Println("Next steps:")
419 if appviewCreated || holdCreated {
420 fmt.Println(" 1. Edit configs if needed, then start services:")
421 } else {
422 fmt.Println(" 1. Start services:")
423 }
424 if state.ScannerEnabled {
425 fmt.Printf(" systemctl start %s / %s / %s\n", naming.Appview(), naming.Hold(), naming.Scanner())
426 } else {
427 fmt.Printf(" systemctl start %s / %s\n", naming.Appview(), naming.Hold())
428 }
429 fmt.Println(" 2. Configure DNS records above")
430
431 return nil
432}
433
434// provisionObjectStorage creates a new Managed Object Storage with a user, access key, and bucket.
435// Returns the state and the secret key separately (only available at creation time).
436func provisionObjectStorage(ctx context.Context, svc *service.Service, zone, s3Name string) (ObjectStorageState, string, error) {
437 // Map compute zone to object storage region (e.g. us-chi1 → us-east-1)
438 region := objectStorageRegion(zone)
439
440 storage, err := svc.CreateManagedObjectStorage(ctx, &request.CreateManagedObjectStorageRequest{
441 Name: s3Name,
442 Region: region,
443 ConfiguredStatus: upcloud.ManagedObjectStorageConfiguredStatusStarted,
444 Networks: []upcloud.ManagedObjectStorageNetwork{
445 {
446 Family: upcloud.IPAddressFamilyIPv4,
447 Name: "public",
448 Type: "public",
449 },
450 },
451 })
452 if err != nil {
453 return ObjectStorageState{}, "", fmt.Errorf("create storage: %w", err)
454 }
455 fmt.Printf(" Created: %s (region: %s)\n", storage.UUID, region)
456
457 // Find endpoint
458 var endpoint string
459 for _, ep := range storage.Endpoints {
460 if ep.DomainName != "" {
461 endpoint = "https://" + ep.DomainName
462 break
463 }
464 }
465
466 // Create user
467 _, err = svc.CreateManagedObjectStorageUser(ctx, &request.CreateManagedObjectStorageUserRequest{
468 ServiceUUID: storage.UUID,
469 Username: s3Name,
470 })
471 if err != nil {
472 return ObjectStorageState{}, "", fmt.Errorf("create user: %w", err)
473 }
474
475 // Attach admin policy
476 err = svc.AttachManagedObjectStorageUserPolicy(ctx, &request.AttachManagedObjectStorageUserPolicyRequest{
477 ServiceUUID: storage.UUID,
478 Username: s3Name,
479 Name: "admin",
480 })
481 if err != nil {
482 return ObjectStorageState{}, "", fmt.Errorf("attach policy: %w", err)
483 }
484
485 // Create access key (secret is only returned here)
486 accessKey, err := svc.CreateManagedObjectStorageUserAccessKey(ctx, &request.CreateManagedObjectStorageUserAccessKeyRequest{
487 ServiceUUID: storage.UUID,
488 Username: s3Name,
489 })
490 if err != nil {
491 return ObjectStorageState{}, "", fmt.Errorf("create access key: %w", err)
492 }
493
494 secretKey := ""
495 if accessKey.SecretAccessKey != nil {
496 secretKey = *accessKey.SecretAccessKey
497 }
498
499 // Create bucket
500 _, err = svc.CreateManagedObjectStorageBucket(ctx, &request.CreateManagedObjectStorageBucketRequest{
501 ServiceUUID: storage.UUID,
502 Name: s3Name,
503 })
504 if err != nil {
505 return ObjectStorageState{}, "", fmt.Errorf("create bucket: %w", err)
506 }
507
508 return ObjectStorageState{
509 UUID: storage.UUID,
510 Endpoint: endpoint,
511 Region: region,
512 Bucket: s3Name,
513 AccessKeyID: accessKey.AccessKeyID,
514 }, secretKey, nil
515}
516
517// objectStorageRegion maps a compute zone to the nearest object storage region.
518func objectStorageRegion(zone string) string {
519 switch {
520 case strings.HasPrefix(zone, "us-"):
521 return "us-east-1"
522 case strings.HasPrefix(zone, "de-"):
523 return "europe-1"
524 case strings.HasPrefix(zone, "fi-"):
525 return "europe-1"
526 case strings.HasPrefix(zone, "nl-"):
527 return "europe-1"
528 case strings.HasPrefix(zone, "es-"):
529 return "europe-1"
530 case strings.HasPrefix(zone, "pl-"):
531 return "europe-1"
532 case strings.HasPrefix(zone, "se-"):
533 return "europe-1"
534 case strings.HasPrefix(zone, "au-"):
535 return "australia-1"
536 case strings.HasPrefix(zone, "sg-"):
537 return "singapore-1"
538 default:
539 return "us-east-1"
540 }
541}
542
543func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, templateUUID, networkUUID, title, userData string) (*ServerState, error) {
544 storageTier := "maxiops"
545 if strings.HasPrefix(strings.ToUpper(cfg.Plan), "DEV-") {
546 storageTier = "standard"
547 }
548
549 // Look up the plan's storage size from the API instead of hardcoding.
550 diskSize := 25 // fallback
551 plans, err := svc.GetPlans(ctx)
552 if err == nil {
553 for _, p := range plans.Plans {
554 if p.Name == cfg.Plan {
555 diskSize = p.StorageSize
556 break
557 }
558 }
559 }
560
561 details, err := svc.CreateServer(ctx, &request.CreateServerRequest{
562 Zone: cfg.Zone,
563 Title: title,
564 Hostname: title,
565 Plan: cfg.Plan,
566 Metadata: upcloud.True,
567 UserData: userData,
568 Firewall: "on",
569 PasswordDelivery: "none",
570 StorageDevices: request.CreateServerStorageDeviceSlice{
571 {
572 Action: "clone",
573 Storage: templateUUID,
574 Title: title + "-disk",
575 Size: diskSize,
576 Tier: storageTier,
577 },
578 },
579 Networking: &request.CreateServerNetworking{
580 Interfaces: request.CreateServerInterfaceSlice{
581 {
582 Index: 1,
583 Type: upcloud.IPAddressAccessPublic,
584 IPAddresses: request.CreateServerIPAddressSlice{
585 {Family: upcloud.IPAddressFamilyIPv4},
586 },
587 },
588 {
589 Index: 2,
590 Type: upcloud.IPAddressAccessPrivate,
591 Network: networkUUID,
592 IPAddresses: request.CreateServerIPAddressSlice{
593 {Family: upcloud.IPAddressFamilyIPv4},
594 },
595 },
596 },
597 },
598 LoginUser: &request.LoginUser{
599 CreatePassword: "no",
600 SSHKeys: request.SSHKeySlice{cfg.SSHPublicKey},
601 },
602 })
603 if err != nil {
604 return nil, err
605 }
606
607 fmt.Printf(" Waiting for server %s to start...\n", details.UUID)
608 details, err = svc.WaitForServerState(ctx, &request.WaitForServerStateRequest{
609 UUID: details.UUID,
610 DesiredState: upcloud.ServerStateStarted,
611 })
612 if err != nil {
613 return nil, fmt.Errorf("wait for server: %w", err)
614 }
615
616 s := &ServerState{UUID: details.UUID}
617 for _, iface := range details.Networking.Interfaces {
618 for _, addr := range iface.IPAddresses {
619 if addr.Family == upcloud.IPAddressFamilyIPv4 {
620 switch iface.Type {
621 case upcloud.IPAddressAccessPublic:
622 s.PublicIP = addr.Address
623 case upcloud.IPAddressAccessPrivate:
624 s.PrivateIP = addr.Address
625 }
626 }
627 }
628 }
629
630 return s, nil
631}
632
633func createFirewallRules(ctx context.Context, svc *service.Service, serverUUID, privateCIDR string) error {
634 networkBase := strings.TrimSuffix(privateCIDR, "/24")
635 networkBase = strings.TrimSuffix(networkBase, ".0")
636
637 return svc.CreateFirewallRules(ctx, &request.CreateFirewallRulesRequest{
638 ServerUUID: serverUUID,
639 FirewallRules: request.FirewallRuleSlice{
640 {
641 Direction: upcloud.FirewallRuleDirectionIn,
642 Action: upcloud.FirewallRuleActionAccept,
643 Family: upcloud.IPAddressFamilyIPv4,
644 Protocol: upcloud.FirewallRuleProtocolTCP,
645 DestinationPortStart: "22",
646 DestinationPortEnd: "22",
647 Position: 1,
648 Comment: "Allow SSH",
649 },
650 {
651 Direction: upcloud.FirewallRuleDirectionIn,
652 Action: upcloud.FirewallRuleActionAccept,
653 Family: upcloud.IPAddressFamilyIPv4,
654 SourceAddressStart: networkBase + ".0",
655 SourceAddressEnd: networkBase + ".255",
656 Position: 2,
657 Comment: "Allow private network",
658 },
659 {
660 Direction: upcloud.FirewallRuleDirectionIn,
661 Action: upcloud.FirewallRuleActionAccept,
662 Family: upcloud.IPAddressFamilyIPv4,
663 Protocol: upcloud.FirewallRuleProtocolUDP,
664 SourcePortStart: "123",
665 SourcePortEnd: "123",
666 Position: 3,
667 Comment: "Allow NTP replies",
668 },
669 {
670 Direction: upcloud.FirewallRuleDirectionIn,
671 Action: upcloud.FirewallRuleActionDrop,
672 Position: 4,
673 Comment: "Drop all other inbound",
674 },
675 },
676 })
677}
678
679func createLoadBalancer(ctx context.Context, svc *service.Service, cfg *InfraConfig, naming Naming, networkUUID, appviewIP, holdIP, holdDomain string) (*upcloud.LoadBalancer, error) {
680 lb, err := svc.CreateLoadBalancer(ctx, &request.CreateLoadBalancerRequest{
681 Name: naming.LBName(),
682 Plan: "essentials",
683 Zone: cfg.Zone,
684 ConfiguredStatus: upcloud.LoadBalancerConfiguredStatusStarted,
685 Networks: []request.LoadBalancerNetwork{
686 {
687 Name: "public",
688 Type: upcloud.LoadBalancerNetworkTypePublic,
689 Family: upcloud.LoadBalancerAddressFamilyIPv4,
690 },
691 {
692 Name: "private",
693 Type: upcloud.LoadBalancerNetworkTypePrivate,
694 Family: upcloud.LoadBalancerAddressFamilyIPv4,
695 UUID: networkUUID,
696 },
697 },
698 Frontends: []request.LoadBalancerFrontend{
699 {
700 Name: "https",
701 Mode: upcloud.LoadBalancerModeHTTP,
702 Port: 443,
703 DefaultBackend: "appview",
704 Networks: []upcloud.LoadBalancerFrontendNetwork{
705 {Name: "public"},
706 },
707 Rules: []request.LoadBalancerFrontendRule{
708 {
709 Name: "set-forwarded-headers",
710 Priority: 1,
711 Matchers: []upcloud.LoadBalancerMatcher{},
712 Actions: []upcloud.LoadBalancerAction{
713 request.NewLoadBalancerSetForwardedHeadersAction(),
714 },
715 },
716 {
717 Name: "route-hold",
718 Priority: 10,
719 Matchers: []upcloud.LoadBalancerMatcher{
720 {
721 Type: upcloud.LoadBalancerMatcherTypeHost,
722 Host: &upcloud.LoadBalancerMatcherHost{
723 Value: holdDomain,
724 },
725 },
726 },
727 Actions: []upcloud.LoadBalancerAction{
728 request.NewLoadBalancerSetForwardedHeadersAction(),
729 {
730 Type: upcloud.LoadBalancerActionTypeUseBackend,
731 UseBackend: &upcloud.LoadBalancerActionUseBackend{
732 Backend: "hold",
733 },
734 },
735 },
736 },
737 },
738 },
739 {
740 Name: "http-redirect",
741 Mode: upcloud.LoadBalancerModeHTTP,
742 Port: 80,
743 DefaultBackend: "appview",
744 Networks: []upcloud.LoadBalancerFrontendNetwork{
745 {Name: "public"},
746 },
747 Rules: []request.LoadBalancerFrontendRule{
748 {
749 Name: "redirect-https",
750 Priority: 10,
751 Matchers: []upcloud.LoadBalancerMatcher{
752 {
753 Type: upcloud.LoadBalancerMatcherTypeSrcPort,
754 SrcPort: &upcloud.LoadBalancerMatcherInteger{
755 Method: upcloud.LoadBalancerIntegerMatcherMethodEqual,
756 Value: 80,
757 },
758 },
759 },
760 Actions: []upcloud.LoadBalancerAction{
761 {
762 Type: upcloud.LoadBalancerActionTypeHTTPRedirect,
763 HTTPRedirect: &upcloud.LoadBalancerActionHTTPRedirect{
764 Scheme: upcloud.LoadBalancerActionHTTPRedirectSchemeHTTPS,
765 },
766 },
767 },
768 },
769 },
770 },
771 },
772 Resolvers: []request.LoadBalancerResolver{},
773 Backends: []request.LoadBalancerBackend{
774 {
775 Name: "appview",
776 Members: []request.LoadBalancerBackendMember{
777 {
778 Name: "appview-1",
779 Type: upcloud.LoadBalancerBackendMemberTypeStatic,
780 IP: appviewIP,
781 Port: 5000,
782 Weight: 100,
783 MaxSessions: 1000,
784 Enabled: true,
785 },
786 },
787 Properties: &upcloud.LoadBalancerBackendProperties{
788 HealthCheckType: upcloud.LoadBalancerHealthCheckTypeHTTP,
789 HealthCheckURL: "/health",
790 },
791 },
792 {
793 Name: "hold",
794 Members: []request.LoadBalancerBackendMember{
795 {
796 Name: "hold-1",
797 Type: upcloud.LoadBalancerBackendMemberTypeStatic,
798 IP: holdIP,
799 Port: 8080,
800 Weight: 100,
801 MaxSessions: 1000,
802 Enabled: true,
803 },
804 },
805 Properties: &upcloud.LoadBalancerBackendProperties{
806 HealthCheckType: upcloud.LoadBalancerHealthCheckTypeHTTP,
807 HealthCheckURL: "/xrpc/_health",
808 },
809 },
810 },
811 })
812 if err != nil {
813 return nil, err
814 }
815
816 return lb, nil
817}
818
819// ensureLBCertificates reconciles TLS certificate bundles on the load balancer.
820// It skips domains that already have a TLS config attached and creates missing ones.
821func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID string, tlsDomains []string) error {
822 lb, err := svc.GetLoadBalancer(ctx, &request.GetLoadBalancerRequest{UUID: lbUUID})
823 if err != nil {
824 return fmt.Errorf("get load balancer: %w", err)
825 }
826
827 // Build set of existing TLS config names on the "https" frontend
828 existing := make(map[string]bool)
829 for _, fe := range lb.Frontends {
830 if fe.Name == "https" {
831 for _, tc := range fe.TLSConfigs {
832 existing[tc.Name] = true
833 }
834 }
835 }
836
837 for _, domain := range tlsDomains {
838 certName := "tls-" + strings.ReplaceAll(domain, ".", "-")
839 if existing[certName] {
840 fmt.Printf(" TLS certificate: %s (exists)\n", domain)
841 continue
842 }
843
844 bundle, err := svc.CreateLoadBalancerCertificateBundle(ctx, &request.CreateLoadBalancerCertificateBundleRequest{
845 Type: upcloud.LoadBalancerCertificateBundleTypeDynamic,
846 Name: certName,
847 KeyType: "ecdsa",
848 Hostnames: []string{domain},
849 })
850 if err != nil {
851 return fmt.Errorf("create TLS cert for %s: %w", domain, err)
852 }
853
854 _, err = svc.CreateLoadBalancerFrontendTLSConfig(ctx, &request.CreateLoadBalancerFrontendTLSConfigRequest{
855 ServiceUUID: lbUUID,
856 FrontendName: "https",
857 Config: request.LoadBalancerFrontendTLSConfig{
858 Name: certName,
859 CertificateBundleUUID: bundle.UUID,
860 },
861 })
862 if err != nil {
863 return fmt.Errorf("attach TLS cert %s to frontend: %w", domain, err)
864 }
865 fmt.Printf(" TLS certificate: %s\n", domain)
866 }
867
868 return nil
869}
870
871// ensureLBForwardedHeaders ensures the "https" frontend has a set_forwarded_headers rule.
872// This makes the LB set X-Forwarded-For, X-Forwarded-Proto, and X-Forwarded-Port headers,
873// overwriting any pre-existing values (prevents spoofing).
874func ensureLBForwardedHeaders(ctx context.Context, svc *service.Service, lbUUID string) error {
875 rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{
876 ServiceUUID: lbUUID,
877 FrontendName: "https",
878 })
879 if err != nil {
880 return fmt.Errorf("get frontend rules: %w", err)
881 }
882
883 for _, r := range rules {
884 if r.Name == "set-forwarded-headers" {
885 // Verify it has the set_forwarded_headers action
886 for _, a := range r.Actions {
887 if a.SetForwardedHeaders != nil {
888 fmt.Println(" Forwarded headers rule: exists and valid")
889 return nil
890 }
891 }
892 // Rule exists but is misconfigured — delete and recreate
893 fmt.Println(" Forwarded headers rule: exists but misconfigured, recreating")
894 if err := svc.DeleteLoadBalancerFrontendRule(ctx, &request.DeleteLoadBalancerFrontendRuleRequest{
895 ServiceUUID: lbUUID,
896 FrontendName: "https",
897 Name: r.Name,
898 }); err != nil {
899 return fmt.Errorf("delete misconfigured forwarded headers rule: %w", err)
900 }
901 break
902 }
903 }
904
905 _, err = svc.CreateLoadBalancerFrontendRule(ctx, &request.CreateLoadBalancerFrontendRuleRequest{
906 ServiceUUID: lbUUID,
907 FrontendName: "https",
908 Rule: request.LoadBalancerFrontendRule{
909 Name: "set-forwarded-headers",
910 Priority: 1,
911 Matchers: []upcloud.LoadBalancerMatcher{},
912 Actions: []upcloud.LoadBalancerAction{
913 request.NewLoadBalancerSetForwardedHeadersAction(),
914 },
915 },
916 })
917 if err != nil {
918 return fmt.Errorf("create forwarded headers rule: %w", err)
919 }
920 fmt.Println(" Forwarded headers rule: created")
921
922 return nil
923}
924
925// ensureLBHoldForwardedHeaders ensures the "route-hold" rule includes a
926// set_forwarded_headers action alongside use_backend. Without this, the LB
927// doesn't set X-Forwarded-For on hold-routed traffic.
928func ensureLBHoldForwardedHeaders(ctx context.Context, svc *service.Service, lbUUID, holdDomain string) error {
929 rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{
930 ServiceUUID: lbUUID,
931 FrontendName: "https",
932 })
933 if err != nil {
934 return fmt.Errorf("get frontend rules: %w", err)
935 }
936
937 for _, r := range rules {
938 if r.Name == "route-hold" {
939 hasForwarded := false
940 for _, a := range r.Actions {
941 if a.SetForwardedHeaders != nil {
942 hasForwarded = true
943 break
944 }
945 }
946 if hasForwarded {
947 fmt.Println(" Route-hold forwarded headers: exists")
948 return nil
949 }
950 // Delete and recreate with both actions
951 fmt.Println(" Route-hold forwarded headers: missing, recreating rule")
952 if err := svc.DeleteLoadBalancerFrontendRule(ctx, &request.DeleteLoadBalancerFrontendRuleRequest{
953 ServiceUUID: lbUUID,
954 FrontendName: "https",
955 Name: r.Name,
956 }); err != nil {
957 return fmt.Errorf("delete route-hold rule: %w", err)
958 }
959 break
960 }
961 }
962
963 _, err = svc.CreateLoadBalancerFrontendRule(ctx, &request.CreateLoadBalancerFrontendRuleRequest{
964 ServiceUUID: lbUUID,
965 FrontendName: "https",
966 Rule: request.LoadBalancerFrontendRule{
967 Name: "route-hold",
968 Priority: 10,
969 Matchers: []upcloud.LoadBalancerMatcher{
970 {
971 Type: upcloud.LoadBalancerMatcherTypeHost,
972 Host: &upcloud.LoadBalancerMatcherHost{
973 Value: holdDomain,
974 },
975 },
976 },
977 Actions: []upcloud.LoadBalancerAction{
978 request.NewLoadBalancerSetForwardedHeadersAction(),
979 {
980 Type: upcloud.LoadBalancerActionTypeUseBackend,
981 UseBackend: &upcloud.LoadBalancerActionUseBackend{
982 Backend: "hold",
983 },
984 },
985 },
986 },
987 })
988 if err != nil {
989 return fmt.Errorf("create route-hold rule: %w", err)
990 }
991 fmt.Println(" Route-hold forwarded headers: created")
992
993 return nil
994}
995
996// ensureLBScannerBlock ensures the "https" frontend has a rule that returns 403
997// for common scanner paths (.php, .asp, .aspx, .jsp, .cgi, .env).
998func ensureLBScannerBlock(ctx context.Context, svc *service.Service, lbUUID string) error {
999 rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{
1000 ServiceUUID: lbUUID,
1001 FrontendName: "https",
1002 })
1003 if err != nil {
1004 return fmt.Errorf("get frontend rules: %w", err)
1005 }
1006
1007 for _, r := range rules {
1008 if r.Name == "block-scanners" {
1009 for _, a := range r.Actions {
1010 if a.HTTPReturn != nil {
1011 fmt.Println(" Scanner block rule: exists and valid")
1012 return nil
1013 }
1014 }
1015 fmt.Println(" Scanner block rule: exists but misconfigured, recreating")
1016 if err := svc.DeleteLoadBalancerFrontendRule(ctx, &request.DeleteLoadBalancerFrontendRuleRequest{
1017 ServiceUUID: lbUUID,
1018 FrontendName: "https",
1019 Name: r.Name,
1020 }); err != nil {
1021 return fmt.Errorf("delete misconfigured scanner block rule: %w", err)
1022 }
1023 break
1024 }
1025 }
1026
1027 ignoreCase := true
1028 _, err = svc.CreateLoadBalancerFrontendRule(ctx, &request.CreateLoadBalancerFrontendRuleRequest{
1029 ServiceUUID: lbUUID,
1030 FrontendName: "https",
1031 Rule: request.LoadBalancerFrontendRule{
1032 Name: "block-scanners",
1033 Priority: 2,
1034 Matchers: []upcloud.LoadBalancerMatcher{
1035 request.NewLoadBalancerPathMatcher(
1036 upcloud.LoadBalancerStringMatcherMethodRegexp,
1037 `\.(php|asp|aspx|jsp|cgi|env)$`,
1038 &ignoreCase,
1039 ),
1040 },
1041 Actions: []upcloud.LoadBalancerAction{
1042 {
1043 Type: upcloud.LoadBalancerActionTypeHTTPReturn,
1044 HTTPReturn: &upcloud.LoadBalancerActionHTTPReturn{
1045 Status: 403,
1046 ContentType: "text/plain",
1047 Payload: base64.StdEncoding.EncodeToString([]byte("Forbidden")),
1048 },
1049 },
1050 },
1051 },
1052 })
1053 if err != nil {
1054 return fmt.Errorf("create scanner block rule: %w", err)
1055 }
1056 fmt.Println(" Scanner block rule: created")
1057
1058 return nil
1059}
1060
1061// lookupObjectStorage discovers details of an existing Managed Object Storage.
1062func lookupObjectStorage(ctx context.Context, svc *service.Service, uuid string) (ObjectStorageState, error) {
1063 storage, err := svc.GetManagedObjectStorage(ctx, &request.GetManagedObjectStorageRequest{
1064 UUID: uuid,
1065 })
1066 if err != nil {
1067 return ObjectStorageState{}, fmt.Errorf("get object storage %s: %w", uuid, err)
1068 }
1069
1070 var endpoint string
1071 for _, ep := range storage.Endpoints {
1072 if ep.DomainName != "" {
1073 endpoint = "https://" + ep.DomainName
1074 break
1075 }
1076 }
1077
1078 var bucket string
1079 buckets, err := svc.GetManagedObjectStorageBucketMetrics(ctx, &request.GetManagedObjectStorageBucketMetricsRequest{
1080 ServiceUUID: uuid,
1081 })
1082 if err == nil {
1083 for _, b := range buckets {
1084 if !b.Deleted {
1085 bucket = b.Name
1086 break
1087 }
1088 }
1089 }
1090
1091 var accessKeyID string
1092 users, err := svc.GetManagedObjectStorageUsers(ctx, &request.GetManagedObjectStorageUsersRequest{
1093 ServiceUUID: uuid,
1094 })
1095 if err == nil {
1096 for _, u := range users {
1097 for _, k := range u.AccessKeys {
1098 if k.Status == "Active" {
1099 accessKeyID = k.AccessKeyID
1100 break
1101 }
1102 }
1103 if accessKeyID != "" {
1104 break
1105 }
1106 }
1107 }
1108
1109 return ObjectStorageState{
1110 UUID: uuid,
1111 Endpoint: endpoint,
1112 Region: storage.Region,
1113 Bucket: bucket,
1114 AccessKeyID: accessKeyID,
1115 }, nil
1116}
1117
1118func findDebianTemplate(ctx context.Context, svc *service.Service) (string, error) {
1119 storages, err := svc.GetStorages(ctx, &request.GetStoragesRequest{
1120 Type: "template",
1121 })
1122 if err != nil {
1123 return "", fmt.Errorf("list templates: %w", err)
1124 }
1125
1126 var debian13, debian12 string
1127 for _, s := range storages.Storages {
1128 title := strings.ToLower(s.Title)
1129 if strings.Contains(title, "debian") {
1130 if strings.Contains(title, "13") || strings.Contains(title, "trixie") {
1131 debian13 = s.UUID
1132 } else if strings.Contains(title, "12") || strings.Contains(title, "bookworm") {
1133 debian12 = s.UUID
1134 }
1135 }
1136 }
1137
1138 if debian13 != "" {
1139 return debian13, nil
1140 }
1141 if debian12 != "" {
1142 fmt.Println(" Debian 13 not available, using Debian 12")
1143 return debian12, nil
1144 }
1145
1146 return "", fmt.Errorf("no Debian template found — check UpCloud template list")
1147}
1148
1149const cloudInitPath = "/var/lib/cloud/instance/scripts/part-001"
1150
1151// syncCloudInit compares a locally-generated cloud-init script against what's
1152// on the server. If they differ (or the remote is missing), it prompts the
1153// user and re-runs the script over SSH.
1154func syncCloudInit(name, ip, localScript string) error {
1155 // Fetch the remote script
1156 remoteScript, err := runSSH(ip, fmt.Sprintf("cat %s 2>/dev/null || echo '__MISSING__'", cloudInitPath), false)
1157 if err != nil {
1158 fmt.Printf(" cloud-init: could not reach %s (%v)\n", name, err)
1159 return nil
1160 }
1161 remoteScript = strings.TrimSpace(remoteScript)
1162
1163 if remoteScript == "__MISSING__" {
1164 fmt.Printf(" cloud-init: not found on %s (server may need initial setup)\n", name)
1165 } else {
1166 localHash := fmt.Sprintf("%x", sha256.Sum256([]byte(strings.TrimSpace(localScript))))
1167 remoteHash := fmt.Sprintf("%x", sha256.Sum256([]byte(remoteScript)))
1168 if localHash == remoteHash {
1169 fmt.Printf(" cloud-init: up to date\n")
1170 return nil
1171 }
1172 fmt.Printf(" cloud-init: differs from local\n")
1173 }
1174
1175 fmt.Printf(" Re-run cloud-init on %s? [Y/n] ", name)
1176 scanner := bufio.NewScanner(os.Stdin)
1177 scanner.Scan()
1178 answer := strings.TrimSpace(strings.ToLower(scanner.Text()))
1179 if answer != "" && answer != "y" && answer != "yes" {
1180 fmt.Printf(" Skipped\n")
1181 // Still update the remote reference so next provision sees an accurate diff
1182 if err := writeRemoteCloudInit(ip, localScript); err != nil {
1183 fmt.Printf(" WARNING: could not update remote cloud-init reference: %v\n", err)
1184 }
1185 return nil
1186 }
1187
1188 // Write the reference file first so next provision can detect real diffs,
1189 // regardless of whether the script execution succeeds or fails.
1190 if err := writeRemoteCloudInit(ip, localScript); err != nil {
1191 fmt.Printf(" WARNING: could not update remote cloud-init reference: %v\n", err)
1192 }
1193
1194 fmt.Printf(" Running cloud-init on %s (%s)... (this may take several minutes)\n", name, ip)
1195 output, err := runSSH(ip, localScript, true)
1196 if err != nil {
1197 fmt.Printf(" ERROR: %v\n", err)
1198 fmt.Printf(" Output:\n%s\n", output)
1199 return fmt.Errorf("cloud-init %s failed", name)
1200 }
1201
1202 fmt.Printf(" %s: cloud-init complete\n", name)
1203 return nil
1204}
1205
1206// generateScannerSecret generates a random 32-byte hex-encoded shared secret
1207// for authenticating scanner-to-hold WebSocket connections.
1208func generateScannerSecret() (string, error) {
1209 b := make([]byte, 32)
1210 if _, err := crypto_rand.Read(b); err != nil {
1211 return "", err
1212 }
1213 return hex.EncodeToString(b), nil
1214}
1215
1216// writeRemoteCloudInit writes the local cloud-init script to the remote server
1217// so that subsequent provision runs can accurately detect real changes.
1218// Uses base64 encoding to avoid heredoc nesting issues (the cloud-init script
1219// itself contains heredocs like CFGEOF and SVCEOF).
1220func writeRemoteCloudInit(ip, script string) error {
1221 encoded := base64.StdEncoding.EncodeToString([]byte(script))
1222 cmd := fmt.Sprintf("mkdir -p $(dirname %s) && echo '%s' | base64 -d > %s", cloudInitPath, encoded, cloudInitPath)
1223 _, err := runSSH(ip, cmd, false)
1224 return err
1225}
1226
1227// waitForSetup polls SSH availability on a newly created server, then waits
1228// for cloud-init to complete before returning.
1229func waitForSetup(ip, name string) error {
1230 fmt.Printf(" %s (%s): waiting for SSH...\n", name, ip)
1231 for i := 0; i < 30; i++ {
1232 _, err := runSSH(ip, "echo ssh_ready", false)
1233 if err == nil {
1234 break
1235 }
1236 if i == 29 {
1237 return fmt.Errorf("SSH not available after 5 minutes on %s (%s)", name, ip)
1238 }
1239 time.Sleep(10 * time.Second)
1240 }
1241
1242 fmt.Printf(" %s: waiting for cloud-init...\n", name)
1243 _, err := runSSH(ip, "cloud-init status --wait 2>/dev/null || true", false)
1244 if err != nil {
1245 return fmt.Errorf("cloud-init wait on %s: %w", name, err)
1246 }
1247 fmt.Printf(" %s: ready\n", name)
1248 return nil
1249}