forked from
tangled.org/core
Monorepo for Tangled
spindle/qemu,nix: use qemu guest agent to execute commands instead of ssh
Signed-off-by: dawn <90008@gaze.systems>
+481
-287
+13
-7
cmd/spindle/main.go
+13
-7
cmd/spindle/main.go
···
9
9
"github.com/urfave/cli/v3"
10
10
tlog "tangled.org/core/log"
11
11
"tangled.org/core/spindle"
12
-
"tangled.org/core/spindle/config"
13
12
"tangled.org/core/spindle/engines/qemu"
14
13
"tangled.org/core/spindle/engines/qemu/bakers"
15
14
)
···
52
51
return &cli.Command{
53
52
Name: "setup-qemu",
54
53
Usage: "downloads and prepares default qemu engine images",
54
+
Flags: []cli.Flag{
55
+
&cli.StringFlag{
56
+
Name: "image-dir",
57
+
Usage: "directory to store images",
58
+
},
59
+
},
55
60
Action: func(ctx context.Context, cmd *cli.Command) error {
56
-
return qemu.SetupDefaultImages(ctx)
61
+
imageDir := cmd.String("image-dir")
62
+
if imageDir == "" {
63
+
imageDir = os.Getenv("SPINDLE_QEMU_PIPELINES_IMAGE_DIR")
64
+
}
65
+
return qemu.SetupDefaultImages(ctx, imageDir)
57
66
},
58
67
}
59
68
}
···
63
72
Name: "bake-image",
64
73
Usage: "(needs root) prepare a downloaded qemu image (e.g. extract kernel/initrd etc.)",
65
74
Action: func(ctx context.Context, cmd *cli.Command) error {
66
-
cfg, err := config.Load(ctx)
67
-
if err != nil {
68
-
return fmt.Errorf("loading config: %w", err)
69
-
}
75
+
imageDir := os.Getenv("SPINDLE_QEMU_PIPELINES_IMAGE_DIR")
70
76
name := cmd.Args().First()
71
77
if name == "" {
72
78
return fmt.Errorf("name argument (e.g. 'alpine') is required")
73
79
}
74
-
return bakers.PrepareImage(ctx, cfg, name)
80
+
return bakers.PrepareImage(ctx, imageDir, name)
75
81
},
76
82
}
77
83
}
+1
-1
flake.nix
+1
-1
flake.nix
+1
-1
nix/modules/spindle.nix
+1
-1
nix/modules/spindle.nix
+1
-1
spindle/config/config.go
+1
-1
spindle/config/config.go
···
46
46
}
47
47
48
48
type QemuPipelines struct {
49
-
ImageDir string `env:"IMAGE_DIR, required"`
49
+
ImageDir string `env:"IMAGE_DIR, required"` // SPINDLE_QEMU_PIPELINES_IMAGE_DIR; also used in cmd/spindle/main.go
50
50
OverlayDir string `env:"OVERLAY_DIR, default="` // where qemu snapshot overlays will live
51
51
DefaultImage string `env:"DEFAULT_IMAGE, default=ubuntu-24.04"`
52
52
EnableKVM bool `env:"ENABLE_KVM, default=true"`
+29
-12
spindle/engines/qemu/bakers/alpine.go
+29
-12
spindle/engines/qemu/bakers/alpine.go
···
37
37
38
38
func (p *AlpineBaker) Prepare(ctx context.Context, dir string) error {
39
39
l := log.FromContext(ctx)
40
-
diskPath := filepath.Join(dir, "disk.qcow2")
40
+
diskPath := DiskPath(dir)
41
+
42
+
// check that we have all the required files
43
+
if _, err := os.Stat(diskPath); os.IsNotExist(err) {
44
+
return fmt.Errorf("missing '%s' in %s", DiskName, dir)
45
+
}
41
46
42
47
nbd, err := p.nbd.Connect(ctx, l, diskPath)
43
48
if err != nil {
···
52
57
}
53
58
defer nbd.Unmount(mntDir)
54
59
55
-
resolvDest := filepath.Join(mntDir, "etc/resolv.conf")
56
-
if err := os.WriteFile(resolvDest, nil, 0644); err != nil {
57
-
return fmt.Errorf("creating resolv.conf mount point: %w", err)
60
+
cleanup, err := bindMountResolvConf(ctx, nbd, mntDir)
61
+
if err != nil {
62
+
return err
58
63
}
59
-
if out, err := exec.CommandContext(ctx, "mount", "--bind", "/etc/resolv.conf", resolvDest).CombinedOutput(); err != nil {
60
-
return fmt.Errorf("bind mount resolv.conf: %s: %w", string(out), err)
64
+
defer cleanup()
65
+
66
+
run := func(command string, args ...string) ([]byte, error) {
67
+
fullArgs := append([]string{mntDir, command}, args...)
68
+
cmd := exec.CommandContext(ctx, "chroot", fullArgs...)
69
+
out, err := cmd.CombinedOutput()
70
+
if err != nil {
71
+
return nil, fmt.Errorf("%s: %s: %w", command, string(out), err)
72
+
}
73
+
return out, nil
61
74
}
62
-
defer nbd.Unmount(resolvDest)
63
75
64
76
l.Info("installing packages...")
65
-
if out, err := exec.CommandContext(ctx, "chroot", mntDir, "/sbin/apk", "add", "--no-cache", "git", "curl").CombinedOutput(); err != nil {
77
+
if out, err := run("/sbin/apk", "add", "--no-cache", "git", "curl", "qemu-guest-agent"); err != nil {
66
78
return fmt.Errorf("installing packages: %s: %w", string(out), err)
67
79
}
68
80
81
+
l.Info("enabling qemu-guest-agent...")
82
+
if out, err := run("/sbin/rc-update", "add", "qemu-guest-agent", "default"); err != nil {
83
+
return fmt.Errorf("enabling qemu-guest-agent: %s: %w", string(out), err)
84
+
}
85
+
69
86
l.Info("extracting kernel and initrd...")
70
87
bootDir := filepath.Join(mntDir, "boot")
71
88
kernelSource := filepath.Join(bootDir, "vmlinuz-virt")
···
77
94
return fmt.Errorf("initrd 'initramfs-virt' not found in %s", bootDir)
78
95
}
79
96
80
-
if err := copyFile(kernelSource, filepath.Join(dir, "kernel")); err != nil {
97
+
if err := copyFile(kernelSource, KernelPath(dir)); err != nil {
81
98
return fmt.Errorf("extracting kernel: %w", err)
82
99
}
83
-
if err := copyFile(initrdSource, filepath.Join(dir, "initrd")); err != nil {
100
+
if err := copyFile(initrdSource, InitrdPath(dir)); err != nil {
84
101
return fmt.Errorf("extracting initrd: %w", err)
85
102
}
86
103
87
-
configPath := filepath.Join(dir, "config.json")
104
+
configPath := ConfigPath(dir)
88
105
metadata := ImageMetadata{
89
-
Cmdline: "root=/dev/vda rootfstype=ext4 rw console=ttyS0 modules=sd-mod,usb-storage,ext4,virtio_pci,virtio_blk",
106
+
Cmdline: "root=/dev/vda rootfstype=ext4 rw console=ttyS0 modules=ext4,virtio_pci,virtio_blk",
90
107
Shell: "/bin/ash",
91
108
}
92
109
b, err := json.MarshalIndent(metadata, "", " ")
+33
-5
spindle/engines/qemu/bakers/baker.go
+33
-5
spindle/engines/qemu/bakers/baker.go
···
9
9
"strings"
10
10
11
11
"tangled.org/core/log"
12
-
"tangled.org/core/spindle/config"
12
+
)
13
+
14
+
const (
15
+
DiskName = "disk"
16
+
KernelName = "kernel"
17
+
InitrdName = "initrd"
18
+
ConfigName = "config.json"
19
+
UserDataName = "user-data.yml"
20
+
SeedISOName = "cloud-init.iso"
13
21
)
14
22
23
+
func DiskPath(dir string) string { return filepath.Join(dir, DiskName) }
24
+
func KernelPath(dir string) string { return filepath.Join(dir, KernelName) }
25
+
func InitrdPath(dir string) string { return filepath.Join(dir, InitrdName) }
26
+
func ConfigPath(dir string) string { return filepath.Join(dir, ConfigName) }
27
+
func UserDataPath(dir string) string { return filepath.Join(dir, UserDataName) }
28
+
func SeedISOPath(dir string) string { return filepath.Join(dir, SeedISOName) }
29
+
15
30
type ImageMetadata struct {
16
31
Cmdline string `json:"cmdline"`
17
32
Shell string `json:"shell"`
···
25
40
nbd = &NBDManager{}
26
41
preparers = map[string]ImageBaker{
27
42
"alpine": &AlpineBaker{nbd: nbd},
28
-
"ubuntu": &UbuntuBaker{},
43
+
"ubuntu": &UbuntuBaker{nbd: nbd},
29
44
}
30
45
)
31
46
32
-
func PrepareImage(ctx context.Context, cfg *config.Config, name string) error {
47
+
func PrepareImage(ctx context.Context, imageDir string, name string) error {
33
48
name = strings.ToLower(name)
34
49
35
-
imageDir := filepath.Join(cfg.QemuPipelines.ImageDir, name)
50
+
imageDir = filepath.Join(imageDir, name)
36
51
if _, err := os.Stat(imageDir); os.IsNotExist(err) {
37
52
return fmt.Errorf("image directory %s does not exist; did you run 'spindle setup'?", imageDir)
38
53
}
···
40
55
l := log.FromContext(ctx)
41
56
l.Info("preparing image", "name", name)
42
57
43
-
diskPath := filepath.Join(imageDir, "disk.qcow2")
58
+
diskPath := DiskPath(imageDir)
44
59
// todo(dawn): this should be configurable
45
60
// cloud-init will expand the partitions itself! so we don't have to do anything else
46
61
l.Info("resizing disk to 8G...", "path", diskPath)
···
57
72
58
73
return nil
59
74
}
75
+
76
+
// bind mount resolve conf for networking
77
+
func bindMountResolvConf(ctx context.Context, nbd *NBDHandle, mntDir string) (func(), error) {
78
+
resolvDest := filepath.Join(mntDir, "etc/resolv.conf")
79
+
_ = os.Remove(resolvDest) // remove if it's a symlink or existing file
80
+
if err := os.WriteFile(resolvDest, nil, 0644); err != nil {
81
+
return nil, fmt.Errorf("creating resolv.conf mount point: %w", err)
82
+
}
83
+
if out, err := exec.CommandContext(ctx, "mount", "--bind", "/etc/resolv.conf", resolvDest).CombinedOutput(); err != nil {
84
+
return nil, fmt.Errorf("bind mount resolv.conf: %s: %w", string(out), err)
85
+
}
86
+
return func() { nbd.Unmount(resolvDest) }, nil
87
+
}
+48
-6
spindle/engines/qemu/bakers/nbd.go
+48
-6
spindle/engines/qemu/bakers/nbd.go
···
6
6
"log/slog"
7
7
"os"
8
8
"os/exec"
9
+
"strings"
9
10
"sync"
10
11
"time"
11
12
)
12
13
13
14
type NBDManager struct {
14
-
mu sync.Mutex
15
+
mu sync.Mutex
16
+
cleanupOnce sync.Once
15
17
}
16
18
17
19
type NBDHandle struct {
···
21
23
}
22
24
23
25
func (n *NBDManager) Connect(ctx context.Context, l *slog.Logger, diskPath string) (*NBDHandle, error) {
26
+
// cleanup in case we have failed state
27
+
n.cleanupOnce.Do(func() {
28
+
n.Cleanup(ctx, l)
29
+
})
30
+
24
31
n.mu.Lock()
25
32
26
-
if _, err := os.Stat("/dev/nbd0"); os.IsNotExist(err) {
33
+
device := "/dev/nbd12" // use something other than nbd0
34
+
35
+
if _, err := os.Stat(device); os.IsNotExist(err) {
27
36
l.Info("nbd module not loaded, trying to load it...")
28
37
if out, err := exec.Command("modprobe", "nbd", "max_part=8").CombinedOutput(); err != nil {
29
38
n.mu.Unlock()
···
32
41
}
33
42
34
43
l.Info("connecting disk via nbd...", "disk", diskPath)
35
-
if out, err := exec.CommandContext(ctx, "qemu-nbd", "--connect=/dev/nbd0", diskPath).CombinedOutput(); err != nil {
44
+
if out, err := exec.CommandContext(ctx, "qemu-nbd", "--connect="+device, diskPath).CombinedOutput(); err != nil {
36
45
n.mu.Unlock()
37
46
return nil, fmt.Errorf("qemu-nbd connect: %s: %w", string(out), err)
38
47
}
···
40
49
waitCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
41
50
defer cancel()
42
51
for {
43
-
if _, err := os.Stat("/dev/nbd0"); err == nil {
52
+
if _, err := os.Stat(device); err == nil {
44
53
break
45
54
}
46
55
select {
47
56
case <-waitCtx.Done():
48
57
n.mu.Unlock()
49
-
return nil, fmt.Errorf("timeout waiting for /dev/nbd0 to appear")
58
+
return nil, fmt.Errorf("timeout waiting for %s to appear", device)
50
59
case <-time.After(50 * time.Millisecond):
51
60
}
52
61
}
53
62
54
-
return &NBDHandle{mgr: n, device: "/dev/nbd0", l: l}, nil
63
+
// sleep 100ms just to make sure...
64
+
time.Sleep(100 * time.Millisecond)
65
+
66
+
return &NBDHandle{mgr: n, device: device, l: l}, nil
55
67
}
56
68
57
69
// mounts the device node specified
···
89
101
h.l.Error("qemu-nbd disconnect failed", "error", err, "output", string(out))
90
102
}
91
103
}
104
+
105
+
func (n *NBDManager) Cleanup(ctx context.Context, l *slog.Logger) {
106
+
l.Info("cleaning up stale nbd state...")
107
+
108
+
// 1. disconnect nbd0
109
+
// do this first to make sure the device is not busy when we try to umount
110
+
_ = exec.CommandContext(ctx, "qemu-nbd", "--disconnect", "/dev/nbd0").Run()
111
+
112
+
// 2. find and unmount stale spindle-mnt mounts
113
+
data, err := os.ReadFile("/proc/mounts")
114
+
if err != nil {
115
+
l.Error("failed to read /proc/mounts", "error", err)
116
+
return
117
+
}
118
+
119
+
for _, line := range strings.Split(string(data), "\n") {
120
+
fields := strings.Fields(line)
121
+
if len(fields) < 2 {
122
+
continue
123
+
}
124
+
mnt := fields[1]
125
+
if strings.Contains(mnt, "spindle-mnt-") {
126
+
l.Info("unmounting stale mount", "path", mnt)
127
+
if out, err := exec.CommandContext(ctx, "umount", "-l", mnt).CombinedOutput(); err != nil {
128
+
l.Error("umount failed", "path", mnt, "error", err, "output", string(out))
129
+
}
130
+
os.RemoveAll(mnt)
131
+
}
132
+
}
133
+
}
+86
-10
spindle/engines/qemu/bakers/ubuntu.go
+86
-10
spindle/engines/qemu/bakers/ubuntu.go
···
5
5
"encoding/json"
6
6
"fmt"
7
7
"os"
8
+
"os/exec"
8
9
"path/filepath"
10
+
"time"
9
11
10
12
"tangled.org/core/log"
11
13
)
12
14
13
-
type UbuntuBaker struct{}
15
+
type UbuntuBaker struct {
16
+
nbd *NBDManager
17
+
}
14
18
15
19
func (p *UbuntuBaker) Prepare(ctx context.Context, dir string) error {
16
20
l := log.FromContext(ctx)
17
21
18
-
_, kerr := os.Stat(filepath.Join(dir, "kernel"))
19
-
_, ierr := os.Stat(filepath.Join(dir, "initrd"))
20
-
// ubuntu preparer assumes kernel and initrd are already downloaded
21
-
if os.IsNotExist(kerr) || os.IsNotExist(ierr) {
22
-
return fmt.Errorf("missing 'kernel' and/or 'initrd' in %s; please use ubuntu cloud images that provide separate kernel/initrd files (see 'unpacked/' on cloud-images.ubuntu.com)", dir)
22
+
// check that we have all the required files
23
+
for _, path := range []string{DiskName, KernelName, InitrdName} {
24
+
if _, err := os.Stat(filepath.Join(dir, path)); os.IsNotExist(err) {
25
+
return fmt.Errorf("missing '%s' in %s", path, dir)
26
+
}
23
27
}
24
28
25
-
configPath := filepath.Join(dir, "config.json")
29
+
diskPath := DiskPath(dir)
30
+
31
+
nbd, err := p.nbd.Connect(ctx, l, diskPath)
32
+
if err != nil {
33
+
return err
34
+
}
35
+
defer nbd.Close()
36
+
37
+
// qemu-nbd might not have scanned partitions yet
38
+
if out, err := exec.CommandContext(ctx, "partprobe", nbd.Device()).CombinedOutput(); err != nil {
39
+
l.Warn("partprobe failed", "error", err, "output", string(out))
40
+
}
41
+
_ = exec.CommandContext(ctx, "udevadm", "settle").Run()
42
+
43
+
// wait for the partition to appear
44
+
partition := nbd.Device() + "p1"
45
+
waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
46
+
defer cancel()
47
+
for {
48
+
if _, err := os.Stat(partition); err == nil {
49
+
break
50
+
}
51
+
select {
52
+
case <-waitCtx.Done():
53
+
return fmt.Errorf("timeout waiting for partition %s to appear", partition)
54
+
case <-time.After(100 * time.Millisecond):
55
+
}
56
+
}
57
+
58
+
mntDir, err := nbd.Mount(ctx, partition)
59
+
if err != nil {
60
+
return err
61
+
}
62
+
defer nbd.Unmount(mntDir)
63
+
64
+
// run := func(command string, args ...string) ([]byte, error) {
65
+
// fullArgs := append([]string{mntDir, "/usr/bin/" + command}, args...)
66
+
// cmd := exec.CommandContext(ctx, "chroot", fullArgs...)
67
+
// cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
68
+
// out, err := cmd.CombinedOutput()
69
+
// if err != nil {
70
+
// return nil, fmt.Errorf("chroot run %s: %s: %w", command, string(out), err)
71
+
// }
72
+
// return out, nil
73
+
// }
74
+
75
+
// cleanup, err := bindMountResolvConf(ctx, nbd, mntDir)
76
+
// if err != nil {
77
+
// return err
78
+
// }
79
+
// defer cleanup()
80
+
81
+
// todo(dawn): i tried doing apt stuff in chroot here but no budge :/
82
+
// i didnt want to hardcode in a qemu-guest-agent either nor replicating apt repo setup...
83
+
// so instead we just write a later user-data file that does it for us at boot.
84
+
85
+
configPath := ConfigPath(dir)
26
86
metadata := ImageMetadata{
27
87
Cmdline: "root=LABEL=cloudimg-rootfs rw console=ttyS0",
28
88
Shell: "/bin/bash",
29
89
}
30
-
if b, err := json.MarshalIndent(metadata, "", " "); err == nil {
31
-
_ = os.WriteFile(configPath, b, 0644)
90
+
b, err := json.MarshalIndent(metadata, "", " ")
91
+
if err != nil {
92
+
return fmt.Errorf("marshaling image metadata: %w", err)
93
+
}
94
+
if err := os.WriteFile(configPath, b, 0644); err != nil {
95
+
return fmt.Errorf("writing image metadata: %w", err)
32
96
}
33
97
34
-
l.Info("ubuntu configuration created")
98
+
// todo(dawn): this should be done while preparing the image...
99
+
userDataPath := UserDataPath(dir)
100
+
userData := `
101
+
packages:
102
+
- qemu-guest-agent
103
+
runcmd:
104
+
- systemctl enable --now qemu-guest-agent
105
+
`
106
+
if err := os.WriteFile(userDataPath, []byte(userData), 0644); err != nil {
107
+
return fmt.Errorf("writing user-data: %w", err)
108
+
}
109
+
110
+
l.Info("ubuntu preparation complete")
35
111
return nil
36
112
}
+9
-6
spindle/engines/qemu/cloudinit.go
+9
-6
spindle/engines/qemu/cloudinit.go
···
5
5
"os"
6
6
"os/exec"
7
7
"path/filepath"
8
+
9
+
"tangled.org/core/spindle/engines/qemu/bakers"
8
10
)
9
11
10
-
func generateSeedISO(dir string, enginePubKey string) error {
12
+
func generateSeedISO(dir string, extraUserData string) error {
11
13
metaData := "instance-id: spindle-vm\nlocal-hostname: spindle\n"
12
-
userData := fmt.Sprintf(`#cloud-config
14
+
userData := `#cloud-config
13
15
users:
14
16
- name: build
15
17
sudo: ALL=(ALL) NOPASSWD:ALL
16
18
shell: /bin/sh
17
-
ssh_authorized_keys:
18
-
- %s
19
-
`, enginePubKey)
19
+
`
20
+
if extraUserData != "" {
21
+
userData += extraUserData
22
+
}
20
23
21
24
metaDataPath := filepath.Join(dir, "meta-data")
22
25
if err := os.WriteFile(metaDataPath, []byte(metaData), 0o644); err != nil {
···
29
32
}
30
33
31
34
// nocloud source expects volid "cidata" with joliet and rock ridge extensions
32
-
cmd := exec.Command("genisoimage", "-output", filepath.Join(dir, "cloud-init.iso"), "-volid", "cidata", "-joliet", "-rock", "meta-data", "user-data")
35
+
cmd := exec.Command("genisoimage", "-output", bakers.SeedISOPath(dir), "-volid", "cidata", "-joliet", "-rock", "meta-data", "user-data")
33
36
cmd.Dir = dir
34
37
cmd.Stdout = os.Stdout
35
38
cmd.Stderr = os.Stderr
+240
-175
spindle/engines/qemu/engine.go
+240
-175
spindle/engines/qemu/engine.go
···
1
1
package qemu
2
2
3
3
import (
4
+
"bufio"
4
5
"context"
5
-
"crypto/ed25519"
6
-
"crypto/rand"
6
+
"encoding/base64"
7
7
"encoding/json"
8
8
"fmt"
9
9
"log/slog"
···
15
15
"time"
16
16
17
17
"github.com/digitalocean/go-qemu/qmp"
18
-
"golang.org/x/crypto/ssh"
19
18
"gopkg.in/yaml.v3"
20
19
21
20
"tangled.org/core/api/tangled"
···
26
25
"tangled.org/core/spindle/models"
27
26
"tangled.org/core/spindle/secrets"
28
27
)
29
-
30
-
type ResolvedImage struct {
31
-
kernel string
32
-
initrd string
33
-
disk string
34
-
cmdline string
35
-
shell string
36
-
}
37
28
38
29
type cleanupFunc func(context.Context) error
39
30
···
63
54
func (s Step) Command() string { return s.command }
64
55
func (s Step) Kind() models.StepKind { return s.kind }
65
56
66
-
type setupSteps []models.Step
67
-
68
-
func (ss *setupSteps) addStep(step models.Step) { *ss = append(*ss, step) }
69
-
70
57
func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) {
71
58
swf := &models.Workflow{}
72
59
var dwf manifestWorkflow
···
86
73
swf.Name = twf.Name
87
74
swf.Environment = dwf.Environment
88
75
89
-
setup := &setupSteps{}
90
-
setup.addStep(models.BuildCloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev))
91
-
92
-
swf.Steps = append(*setup, swf.Steps...)
76
+
setup := []models.Step{models.BuildCloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev)}
77
+
swf.Steps = append(setup, swf.Steps...)
93
78
94
79
img, err := e.resolveImage(dwf.Image)
95
80
if err != nil {
96
81
return nil, err
97
82
}
98
83
99
-
swf.Data = vmState{
100
-
kernel: img.kernel,
101
-
initrd: img.initrd,
102
-
disk: img.disk,
103
-
cmdline: img.cmdline,
104
-
shell: img.shell,
105
-
}
106
-
84
+
swf.Data = vmState{img: img}
107
85
return swf, nil
108
86
}
109
87
110
-
// discover and resolve kernel, initrd, and disk from an image subfolder
88
+
// discover and resolve kernel, initrd, and disk and other config from an image subfolder
111
89
func (e *Engine) resolveImage(name string) (ResolvedImage, error) {
112
90
var img ResolvedImage
113
91
if name == "" {
···
118
96
}
119
97
120
98
imageDir := filepath.Join(e.cfg.QemuPipelines.ImageDir, name)
121
-
kernelPath := filepath.Join(imageDir, "kernel")
122
-
initrdPath := filepath.Join(imageDir, "initrd")
123
-
diskPath := filepath.Join(imageDir, "disk.qcow2")
124
-
configPath := filepath.Join(imageDir, "config.json")
99
+
kernelPath := bakers.KernelPath(imageDir)
100
+
initrdPath := bakers.InitrdPath(imageDir)
101
+
diskPath := bakers.DiskPath(imageDir)
102
+
configPath := bakers.ConfigPath(imageDir)
125
103
126
104
if _, err := os.Stat(diskPath); err == nil {
127
105
img.disk = diskPath
···
143
121
}
144
122
}
145
123
}
124
+
if b, err := os.ReadFile(filepath.Join(imageDir, bakers.UserDataName)); err == nil {
125
+
img.userData = string(b)
126
+
}
146
127
147
128
if img.disk == "" {
148
-
return img, fmt.Errorf("missing 'disk.qcow2' in %s", imageDir)
129
+
return img, fmt.Errorf("missing '%s' in %s", bakers.DiskName, imageDir)
149
130
}
150
131
if img.kernel != "" && (img.initrd == "" || img.cmdline == "") {
151
132
return img, fmt.Errorf("kernel requires initrd and cmdline, but 'initrd' and/or 'cmdline' is missing for %s", name)
···
160
141
l := e.l.With("workflow", wid)
161
142
l.Info("setting up qemu workflow")
162
143
163
-
setupStep := Step{name: "qemu vm setup", kind: models.StepKindSystem}
164
-
setupStepIdx := -1
144
+
const setupStepIdx = -1
145
+
setupStep := Step{name: "QEMU VM setup", kind: models.StepKindSystem}
165
146
166
147
wfLogger.ControlWriter(setupStepIdx, setupStep, models.StepStatusStart).Write([]byte{0})
167
148
defer wfLogger.ControlWriter(setupStepIdx, setupStep, models.StepStatusEnd).Write([]byte{0})
168
149
169
-
// generate ed25519 keypair for the engine to ssh into the guest
170
-
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
171
-
if err != nil {
172
-
return fmt.Errorf("generating keypair: %w", err)
173
-
}
174
-
175
-
sshSigner, err := ssh.NewSignerFromKey(privKey)
176
-
if err != nil {
177
-
return err
178
-
}
179
-
sshPubKey, err := ssh.NewPublicKey(pubKey)
180
-
if err != nil {
181
-
return err
182
-
}
183
-
enginePubKeyStr := string(ssh.MarshalAuthorizedKey(sshPubKey))
184
-
185
-
// the tempdir is configurable since some systems may have tmpfs as /tmp,
186
-
// which is not ideal if a workflow uses a lot of space.
150
+
// some systems have tmpfs at /tmp which is not ideal for large workloads
187
151
targetTempDir := e.cfg.QemuPipelines.OverlayDir
188
152
if targetTempDir == "" {
189
153
targetTempDir = os.TempDir()
···
198
162
})
199
163
200
164
state := wf.Data.(vmState)
165
+
img := state.img
201
166
202
-
// generate nocloud seed iso containing engine's authorized_keys
203
-
if err := generateSeedISO(tempDir, enginePubKeyStr); err != nil {
167
+
if err := generateSeedISO(tempDir, img.userData); err != nil {
204
168
return fmt.Errorf("generating seed iso: %w", err)
205
169
}
206
170
207
-
// note: there is an inherent TOCTOU race here between releasing the port
208
-
// and qemu binding it. nothing we can do about this in userspace without
209
-
// privilege, but in practice it's rarely an issue.
210
-
nl, err := net.Listen("tcp", "127.0.0.1:0")
211
-
if err != nil {
212
-
return fmt.Errorf("finding free port: %w", err)
213
-
}
214
-
sshPort := nl.Addr().(*net.TCPAddr).Port
215
-
nl.Close()
216
-
217
171
qmpSock := filepath.Join(tempDir, "qmp.sock")
172
+
qgaSock := filepath.Join(tempDir, "qga.sock")
173
+
218
174
// todo(dawn): ideally would be nice if we used qemu with the microvm enabled here...
219
175
// but that is not compatible with cloud-init since it expects real hw enumeration...
220
176
// and we would not be able to use standard cloud images, which is kind of annoying.
···
227
183
// todo(dawn): ideally probably have "tiers" and let the spindle concile the tier using
228
184
// what the user wants and what the user has exposed to them by the spindle operator?
229
185
"-m", e.cfg.QemuPipelines.Memory, "-smp", fmt.Sprintf("%d", e.cfg.QemuPipelines.SMP),
230
-
"-display", "none", "-nodefaults", "-no-user-config",
186
+
"-display", "none", "-monitor", "none", "-nodefaults", "-no-user-config",
231
187
// use snapshot=on to do copy-on-write without having us manage qcow overlays manually
232
-
"-drive", fmt.Sprintf("file=%s,media=disk,snapshot=on,if=virtio", state.disk),
233
-
"-drive", fmt.Sprintf("file=%s,media=cdrom", filepath.Join(tempDir, "cloud-init.iso")),
234
-
"-netdev", fmt.Sprintf("user,id=net0,hostfwd=tcp:127.0.0.1:%d-:22", sshPort),
188
+
"-drive", fmt.Sprintf("file=%s,media=disk,snapshot=on,if=virtio", img.disk),
189
+
"-drive", fmt.Sprintf("file=%s,media=cdrom", bakers.SeedISOPath(tempDir)),
190
+
"-netdev", "user,id=net0",
235
191
"-device", "virtio-net-pci,netdev=net0",
236
192
"-qmp", fmt.Sprintf("unix:%s,server,nowait", qmpSock),
237
-
"-monitor", "none",
193
+
"-chardev", fmt.Sprintf("socket,path=%s,server,nowait,id=qga0", qgaSock),
194
+
"-device", "virtio-serial",
195
+
"-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0",
238
196
}
239
197
240
198
if e.cfg.Server.Dev {
···
245
203
246
204
// support booting using qemu bios still, but otherwise we make it faster!
247
205
// incase someone wants to do this for whatever reason...
248
-
if state.kernel != "" {
249
-
argv = append(argv, "-kernel", state.kernel)
250
-
if state.initrd != "" {
251
-
argv = append(argv, "-initrd", state.initrd)
206
+
if img.kernel != "" {
207
+
argv = append(argv, "-kernel", img.kernel)
208
+
if img.initrd != "" {
209
+
argv = append(argv, "-initrd", img.initrd)
252
210
}
253
-
if state.cmdline != "" {
254
-
argv = append(argv, "-append", state.cmdline)
211
+
if img.cmdline != "" {
212
+
argv = append(argv, "-append", img.cmdline)
255
213
}
256
214
} else {
257
-
// if we are booting with bios, we need to tell it to boot from the disk
215
+
// booting with bios: explicitly select disk
258
216
argv = append(argv, "-boot", "order=c")
259
217
}
260
218
···
276
234
qemuCmd.Stdout = os.Stdout
277
235
qemuCmd.Stderr = os.Stderr
278
236
279
-
startBoot := time.Now()
237
+
startedBootAt := time.Now()
280
238
if err := qemuCmd.Start(); err != nil {
281
239
return fmt.Errorf("starting qemu: %w", err)
282
240
}
283
241
284
242
var mon *qmp.SocketMonitor
243
+
244
+
// cleanup qemu if we fail to setup at any point below
245
+
setupOk := false
246
+
defer func() {
247
+
if !setupOk {
248
+
_ = qemuCmd.Process.Kill()
249
+
if mon != nil {
250
+
_ = mon.Disconnect()
251
+
}
252
+
}
253
+
}()
254
+
285
255
qmpCtx, cancelQmp := context.WithTimeout(ctx, 10*time.Second)
286
256
defer cancelQmp()
287
257
288
-
// retry qmp connect until the emulator is ready with its unix socket
258
+
// wait for qmp to be ready
289
259
for {
290
260
mon, err = qmp.NewSocketMonitor("unix", qmpSock, 2*time.Second)
291
261
if err == nil {
···
295
265
}
296
266
select {
297
267
case <-qmpCtx.Done():
298
-
_ = qemuCmd.Process.Kill()
299
268
return fmt.Errorf("qmp connect timeout: %w", err)
300
269
case <-time.After(10 * time.Millisecond):
301
270
}
302
271
}
303
272
304
-
// check if the guest is running
305
-
raw, err := mon.Run([]byte(`{"execute":"query-status"}`))
273
+
status, err := e.qmpQueryStatus(mon)
306
274
if err != nil {
307
-
_ = qemuCmd.Process.Kill()
308
-
_ = mon.Disconnect()
309
-
return fmt.Errorf("qmp query-status failed: %w", err)
275
+
return err
310
276
}
311
-
var resp map[string]any
312
-
if err := json.Unmarshal(raw, &resp); err != nil {
313
-
_ = qemuCmd.Process.Kill()
314
-
_ = mon.Disconnect()
315
-
return fmt.Errorf("qmp query-status parse failed: %w", err)
316
-
}
317
-
status, _ := resp["return"].(map[string]any)["status"].(string)
277
+
318
278
l.Info("qemu guest status", "status", status)
319
279
if status != "running" {
320
-
_ = qemuCmd.Process.Kill()
321
-
_ = mon.Disconnect()
322
280
return fmt.Errorf("qemu guest not running (status: %s)", status)
323
281
}
324
282
325
-
sshConfig := &ssh.ClientConfig{
326
-
User: "build",
327
-
Auth: []ssh.AuthMethod{ssh.PublicKeys(sshSigner)},
328
-
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
329
-
Timeout: 5 * time.Second,
330
-
}
331
-
332
-
bootCtx, cancelBoot := context.WithTimeout(ctx, 90*time.Second)
333
-
defer cancelBoot()
283
+
qgaCtx, cancelQga := context.WithTimeout(ctx, time.Minute)
284
+
defer cancelQga()
334
285
335
-
// backoff until the guest finishes booting and starts sshd
336
-
var sshClient *ssh.Client
286
+
// wait for guest agent to be ready (aka boot)
337
287
for {
338
-
sshClient, err = ssh.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", sshPort), sshConfig)
339
-
if err == nil {
340
-
bootDuration := time.Since(startBoot).Round(time.Millisecond)
341
-
l.Debug("vm booted and ssh ready", "elapsed", bootDuration)
342
-
break
288
+
if _, err := os.Stat(qgaSock); err == nil {
289
+
pingCtx, cancelPing := context.WithTimeout(qgaCtx, 5*time.Second)
290
+
err = e.qgaGuestPing(pingCtx, qgaSock)
291
+
cancelPing()
292
+
if err == nil {
293
+
l.Info("vm booted and guest-agent ready", "elapsed", time.Since(startedBootAt).Round(time.Millisecond))
294
+
break
295
+
}
296
+
l.Debug("qga guest-ping failed", "error", err)
297
+
} else {
298
+
l.Debug("qga socket not found yet", "path", qgaSock)
343
299
}
300
+
344
301
select {
345
-
case <-bootCtx.Done():
346
-
_ = qemuCmd.Process.Kill()
347
-
_ = mon.Disconnect()
348
-
return ErrBootTimeout
302
+
case <-qgaCtx.Done():
303
+
return fmt.Errorf("qga connect timeout: %w", err)
349
304
case <-time.After(100 * time.Millisecond):
350
305
}
351
306
}
352
307
353
308
e.registerCleanup(wid, func(ctx context.Context) error {
354
-
_ = sshClient.Close()
355
-
356
309
// graceful powerdown so guest can sync filesystem
357
-
_, _ = mon.Run([]byte(`{"execute": "system_powerdown"}`))
310
+
if err := e.qmpSystemPowerdown(mon); err != nil {
311
+
l.Error("failed to powerdown qemu guest", "workflow", wid, "error", err)
312
+
}
358
313
359
314
done := make(chan error, 1)
360
-
go func() {
361
-
done <- qemuCmd.Wait()
362
-
}()
315
+
go func() { done <- qemuCmd.Wait() }()
316
+
363
317
select {
364
318
case <-done:
365
-
case <-time.After(time.Second):
319
+
case <-time.After(5 * time.Second):
366
320
_ = qemuCmd.Process.Kill()
367
-
// drain Wait after kill to avoid zombie
368
-
<-done
321
+
<-done // drain to avoid zombie
369
322
}
370
323
371
324
_ = mon.Disconnect()
···
373
326
})
374
327
375
328
wf.Data = vmState{
376
-
process: qemuCmd.Process,
377
-
qmpMon: mon,
378
-
sshClient: sshClient,
379
-
sshPort: sshPort,
380
-
tempDir: tempDir,
381
-
kernel: state.kernel,
382
-
initrd: state.initrd,
383
-
disk: state.disk,
384
-
shell: state.shell,
329
+
process: qemuCmd.Process,
330
+
qmpMon: mon,
331
+
qgaPath: qgaSock,
332
+
tempDir: tempDir,
333
+
img: img,
385
334
}
386
335
336
+
setupOk = true
387
337
return nil
388
338
}
389
339
390
-
func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error {
391
-
state := w.Data.(vmState)
392
-
step := w.Steps[idx]
340
+
func (e *Engine) qmpRun(mon *qmp.SocketMonitor, command qmp.Command) ([]byte, error) {
341
+
b, err := json.Marshal(command)
342
+
if err != nil {
343
+
return nil, err
344
+
}
345
+
return mon.Run(b)
346
+
}
393
347
394
-
var envStr string
395
-
appendEnv := func(k, v string) {
396
-
// we pass the env vars to shell instead of through ssh
397
-
// (we would need to make sshd config have AcceptEnv in seed iso but that kinda sucks
398
-
// because that would reload sshd during boot, which is not ideal, so we can just do this.)
399
-
// todo(dawn): make images be prepared with sshd configured
400
-
envStr += fmt.Sprintf("%s=%s ", k, shellescape(v))
348
+
// sends a command to the qemu guest agent and returns the response
349
+
func (e *Engine) qgaRun(ctx context.Context, sock string, command qmp.Command) ([]byte, error) {
350
+
b, err := json.Marshal(command)
351
+
if err != nil {
352
+
return nil, err
401
353
}
402
354
403
-
for k, v := range w.Environment {
404
-
appendEnv(k, v)
355
+
conn, err := (&net.Dialer{}).DialContext(ctx, "unix", sock)
356
+
if err != nil {
357
+
return nil, err
405
358
}
406
-
for _, s := range secrets {
407
-
appendEnv(s.Key, s.Value)
359
+
defer conn.Close()
360
+
361
+
if dl, ok := ctx.Deadline(); ok {
362
+
_ = conn.SetDeadline(dl)
408
363
}
409
-
if s, ok := step.(Step); ok {
410
-
for k, v := range s.environment {
411
-
appendEnv(k, v)
412
-
}
364
+
365
+
if _, err := conn.Write(append(b, '\n')); err != nil {
366
+
return nil, err
413
367
}
414
368
415
-
session, err := state.sshClient.NewSession()
369
+
return bufio.NewReader(conn).ReadBytes('\n')
370
+
}
371
+
372
+
// query the qemu guest status
373
+
func (e *Engine) qmpQueryStatus(mon *qmp.SocketMonitor) (string, error) {
374
+
raw, err := e.qmpRun(mon, qmp.Command{Execute: "query-status"})
416
375
if err != nil {
417
-
return fmt.Errorf("ssh new session: %w", err)
376
+
return "", fmt.Errorf("qmp query-status failed: %w", err)
418
377
}
419
-
defer session.Close()
420
378
421
-
stdoutPipe, err := session.StdoutPipe()
379
+
var resp struct {
380
+
Return struct {
381
+
Status string `json:"status"`
382
+
} `json:"return"`
383
+
}
384
+
if err := json.Unmarshal(raw, &resp); err != nil {
385
+
return "", fmt.Errorf("qmp query-status parse: %w", err)
386
+
}
387
+
return resp.Return.Status, nil
388
+
}
389
+
390
+
// send a command to qemu to powerdown the guest gracefully
391
+
func (e *Engine) qmpSystemPowerdown(mon *qmp.SocketMonitor) error {
392
+
_, err := e.qmpRun(mon, qmp.Command{Execute: "system_powerdown"})
393
+
return err
394
+
}
395
+
396
+
// ping the guest agent to see if it's ready
397
+
func (e *Engine) qgaGuestPing(ctx context.Context, sock string) error {
398
+
_, err := e.qgaRun(ctx, sock, qmp.Command{Execute: "guest-ping"})
399
+
return err
400
+
}
401
+
402
+
// execute a command on the guest and return its pid
403
+
func (e *Engine) qgaGuestExec(ctx context.Context, sock string, shell string, command string, env []string) (int, error) {
404
+
cmdArgs := []string{shell, "-c", command}
405
+
raw, err := e.qgaRun(ctx, sock, qmp.Command{
406
+
Execute: "guest-exec",
407
+
Args: map[string]any{
408
+
"path": shell,
409
+
"arg": cmdArgs[1:],
410
+
"env": env,
411
+
"capture-output": true,
412
+
},
413
+
})
422
414
if err != nil {
423
-
return fmt.Errorf("ssh stdout pipe: %w", err)
415
+
return 0, fmt.Errorf("qga guest-exec: %w", err)
416
+
}
417
+
418
+
var resp struct {
419
+
Return struct {
420
+
Pid int `json:"pid"`
421
+
} `json:"return"`
424
422
}
425
-
stderrPipe, err := session.StderrPipe()
423
+
if err := json.Unmarshal(raw, &resp); err != nil {
424
+
return 0, fmt.Errorf("qga guest-exec parse: %w", err)
425
+
}
426
+
return resp.Return.Pid, nil
427
+
}
428
+
429
+
type guestExecStatus struct {
430
+
Exited bool
431
+
ExitCode int
432
+
// these are since the last time we asked for status
433
+
OutData []byte
434
+
ErrData []byte
435
+
}
436
+
437
+
// get the status of a command executed with guest-exec
438
+
func (e *Engine) qgaGuestExecStatus(ctx context.Context, sock string, pid int) (guestExecStatus, error) {
439
+
var status guestExecStatus
440
+
raw, err := e.qgaRun(ctx, sock, qmp.Command{
441
+
Execute: "guest-exec-status",
442
+
Args: map[string]any{"pid": pid},
443
+
})
426
444
if err != nil {
427
-
return fmt.Errorf("ssh stderr pipe: %w", err)
445
+
return status, fmt.Errorf("qga guest-exec-status: %w", err)
428
446
}
429
447
430
-
tailDone := make(chan error, 1)
431
-
go func() {
432
-
tailDone <- streamLogs(ctx, stdoutPipe, stderrPipe, idx, wfLogger)
433
-
}()
448
+
var resp struct {
449
+
Return struct {
450
+
Exited bool `json:"exited"`
451
+
ExitCode int `json:"exitcode"`
452
+
OutData string `json:"out-data"`
453
+
ErrData string `json:"err-data"`
454
+
} `json:"return"`
455
+
}
456
+
if err := json.Unmarshal(raw, &resp); err != nil {
457
+
return status, fmt.Errorf("qga guest-exec-status parse: %w", err)
458
+
}
459
+
460
+
status.Exited = resp.Return.Exited
461
+
status.ExitCode = resp.Return.ExitCode
462
+
if resp.Return.OutData != "" {
463
+
status.OutData, _ = base64.StdEncoding.DecodeString(resp.Return.OutData)
464
+
}
465
+
if resp.Return.ErrData != "" {
466
+
status.ErrData, _ = base64.StdEncoding.DecodeString(resp.Return.ErrData)
467
+
}
468
+
469
+
return status, nil
470
+
}
471
+
472
+
func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error {
473
+
state := w.Data.(vmState)
474
+
step := w.Steps[idx]
475
+
476
+
env := make([]string, 0, len(w.Environment)+len(secrets))
477
+
for k, v := range w.Environment {
478
+
env = append(env, k+"="+v)
479
+
}
480
+
for _, s := range secrets {
481
+
env = append(env, s.Key+"="+s.Value)
482
+
}
483
+
if s, ok := step.(Step); ok {
484
+
for k, v := range s.environment {
485
+
env = append(env, k+"="+v)
486
+
}
487
+
}
434
488
435
-
// execute with the image's specified shell
436
-
cmd := fmt.Sprintf("%s%s -c %s", envStr, state.shell, shellescape(step.Command()))
437
-
if err := session.Start(cmd); err != nil {
438
-
return fmt.Errorf("session start: %w", err)
489
+
pid, err := e.qgaGuestExec(ctx, state.qgaPath, state.img.shell, step.Command(), env)
490
+
if err != nil {
491
+
return err
439
492
}
440
493
441
-
select {
442
-
case err := <-tailDone:
494
+
for {
495
+
status, err := e.qgaGuestExecStatus(ctx, state.qgaPath, pid)
443
496
if err != nil {
444
-
e.l.Warn("log streaming error", "workflow", wid, "step", idx, "error", err)
497
+
return err
498
+
}
499
+
500
+
if len(status.OutData) > 0 {
501
+
_, _ = wfLogger.DataWriter(idx, "stdout").Write(status.OutData)
502
+
}
503
+
if len(status.ErrData) > 0 {
504
+
_, _ = wfLogger.DataWriter(idx, "stderr").Write(status.ErrData)
505
+
}
506
+
507
+
if status.Exited {
508
+
if status.ExitCode != 0 {
509
+
return engine.ErrWorkflowFailed
510
+
}
511
+
break
445
512
}
446
-
case <-ctx.Done():
447
-
_ = session.Signal(ssh.SIGKILL)
448
-
<-tailDone
449
-
return engine.ErrTimedOut
450
-
}
451
513
452
-
if err := session.Wait(); err != nil {
453
-
return engine.ErrWorkflowFailed
514
+
select {
515
+
case <-ctx.Done():
516
+
return engine.ErrTimedOut
517
+
case <-time.After(100 * time.Millisecond):
518
+
}
454
519
}
455
520
return nil
456
521
}
-35
spindle/engines/qemu/logs.go
-35
spindle/engines/qemu/logs.go
···
1
-
package qemu
2
-
3
-
import (
4
-
"context"
5
-
"io"
6
-
"strings"
7
-
8
-
"golang.org/x/sync/errgroup"
9
-
"tangled.org/core/spindle/models"
10
-
)
11
-
12
-
// shellescape safely escapes a string for bash -c
13
-
func shellescape(str string) string {
14
-
return "'" + strings.ReplaceAll(str, "'", `'"'"'`) + "'"
15
-
}
16
-
17
-
// streamLogs copies stdout and stderr from the SSH session to the workflow logger
18
-
func streamLogs(ctx context.Context, stdoutPipe io.Reader, stderrPipe io.Reader, idx int, wfLogger models.WorkflowLogger) error {
19
-
if wfLogger == nil {
20
-
return nil
21
-
}
22
-
23
-
g, _ := errgroup.WithContext(ctx)
24
-
25
-
g.Go(func() error {
26
-
_, err := io.Copy(wfLogger.DataWriter(idx, "stdout"), stdoutPipe)
27
-
return err
28
-
})
29
-
g.Go(func() error {
30
-
_, err := io.Copy(wfLogger.DataWriter(idx, "stderr"), stderrPipe)
31
-
return err
32
-
})
33
-
34
-
return g.Wait()
35
-
}
+14
-11
spindle/engines/qemu/models.go
+14
-11
spindle/engines/qemu/models.go
···
4
4
"os"
5
5
6
6
"github.com/digitalocean/go-qemu/qmp"
7
-
"golang.org/x/crypto/ssh"
8
7
)
9
8
10
9
type vmState struct {
11
-
process *os.Process
12
-
qmpMon *qmp.SocketMonitor
13
-
sshClient *ssh.Client
14
-
sshPort int
15
-
tempDir string
10
+
process *os.Process
11
+
qmpMon *qmp.SocketMonitor
12
+
qgaPath string
13
+
tempDir string
14
+
15
+
img ResolvedImage
16
+
}
16
17
17
-
kernel string
18
-
initrd string
19
-
disk string
20
-
cmdline string
21
-
shell string
18
+
type ResolvedImage struct {
19
+
kernel string
20
+
initrd string
21
+
disk string
22
+
cmdline string // kernel command line
23
+
shell string // shell to use for workflow steps
24
+
userData string // extra cloud-init user-data
22
25
}
23
26
24
27
type manifestWorkflow struct {
+6
-17
spindle/engines/qemu/setup.go
+6
-17
spindle/engines/qemu/setup.go
···
10
10
11
11
"golang.org/x/sync/errgroup"
12
12
"tangled.org/core/log"
13
-
"tangled.org/core/spindle/config"
14
13
"tangled.org/core/spindle/engines/qemu/bakers"
15
14
)
16
15
···
73
72
return os.Rename(tmp, dest)
74
73
}
75
74
76
-
func SetupDefaultImages(ctx context.Context) error {
77
-
cfg, err := config.Load(ctx)
78
-
if err != nil {
79
-
return fmt.Errorf("loading config: %w", err)
80
-
}
81
-
82
-
imageDir := cfg.QemuPipelines.ImageDir
75
+
func SetupDefaultImages(ctx context.Context, imageDir string) error {
83
76
if imageDir == "" {
84
-
return fmt.Errorf("SPINDLE_QEMU_PIPELINES_IMAGE_DIR must be set")
77
+
return fmt.Errorf("imageDir must be set")
85
78
}
86
79
87
80
l := log.FromContext(ctx)
···
101
94
return err
102
95
}
103
96
104
-
diskPath := filepath.Join(dir, "disk.qcow2")
105
-
if _, err := os.Stat(diskPath); err == nil {
106
-
l.Info("image already exists, skipping", "name", name)
107
-
return nil
108
-
}
97
+
diskPath := bakers.DiskPath(dir)
109
98
110
99
l.Info("downloading image...", "name", name)
111
100
if err := downloadFile(ctx, img.disk, diskPath); err != nil {
···
113
102
}
114
103
115
104
if img.kernel != "" {
116
-
if err := downloadFile(ctx, img.kernel, filepath.Join(dir, "kernel")); err != nil {
105
+
if err := downloadFile(ctx, img.kernel, bakers.KernelPath(dir)); err != nil {
117
106
return fmt.Errorf("downloading %s kernel: %w", name, err)
118
107
}
119
108
}
120
109
121
110
if img.initrd != "" {
122
-
if err := downloadFile(ctx, img.initrd, filepath.Join(dir, "initrd")); err != nil {
111
+
if err := downloadFile(ctx, img.initrd, bakers.InitrdPath(dir)); err != nil {
123
112
return fmt.Errorf("downloading %s initrd: %w", name, err)
124
113
}
125
114
}
126
115
127
-
if err := bakers.PrepareImage(ctx, cfg, name); err != nil {
116
+
if err := bakers.PrepareImage(ctx, imageDir, name); err != nil {
128
117
return fmt.Errorf("preparing image %s: %w", name, err)
129
118
}
130
119