this repo has no description
feat(toolbox): vendor artifacts into internal registry
+349
+1
toolbox/cmd/root.go
+1
toolbox/cmd/root.go
+82
toolbox/cmd/vendor.go
+82
toolbox/cmd/vendor.go
···
1
+
package cmd
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"os"
7
+
"os/exec"
8
+
9
+
"github.com/spf13/cobra"
10
+
11
+
"github.com/khuedoan/cloudlab/toolbox/internal/cluster"
12
+
"github.com/khuedoan/cloudlab/toolbox/internal/vendors"
13
+
)
14
+
15
+
var vendorCmd = &cobra.Command{
16
+
Use: "vendor",
17
+
Args: cobra.NoArgs,
18
+
Short: "Vendor charts and images from settings.yaml into the in-cluster registry",
19
+
PreRunE: func(_ *cobra.Command, _ []string) error {
20
+
if err := validateClusterFlags(); err != nil {
21
+
return err
22
+
}
23
+
if settingsFile == "" {
24
+
return fmt.Errorf("--settings is required")
25
+
}
26
+
for _, name := range []string{"helm", "oras"} {
27
+
if _, err := exec.LookPath(name); err != nil {
28
+
return fmt.Errorf("find %s CLI: %w", name, err)
29
+
}
30
+
}
31
+
return nil
32
+
},
33
+
RunE: runSync,
34
+
}
35
+
36
+
func init() {
37
+
vendorCmd.Flags().StringVar(&settingsFile, "settings", "", "Path to settings YAML file")
38
+
}
39
+
40
+
func runSync(cmd *cobra.Command, _ []string) error {
41
+
entries, err := vendors.LoadVendors(settingsFile)
42
+
if err != nil {
43
+
return err
44
+
}
45
+
46
+
connectCtx, cancel := context.WithTimeout(cmd.Context(), connectTimeout)
47
+
defer cancel()
48
+
49
+
hostAddr, err := cluster.LoadHost(hostsFile, host)
50
+
if err != nil {
51
+
return fmt.Errorf("load host: %w", err)
52
+
}
53
+
54
+
conn, err := cluster.Connect(cluster.SSHConfig{
55
+
Host: hostAddr,
56
+
User: sshUser,
57
+
KeyPath: sshKey,
58
+
KnownHostsPath: sshKnownHosts,
59
+
Timeout: connectTimeout,
60
+
})
61
+
if err != nil {
62
+
return fmt.Errorf("connect to cluster: %w", err)
63
+
}
64
+
defer conn.Close()
65
+
66
+
tunnel, err := conn.Forward(connectCtx, cluster.ServiceConfig{
67
+
Namespace: registryNamespace,
68
+
Name: registryService,
69
+
Port: registryPort,
70
+
})
71
+
if err != nil {
72
+
return fmt.Errorf("forward registry: %w", err)
73
+
}
74
+
75
+
workdir, err := os.MkdirTemp("", "toolbox-vendor-*")
76
+
if err != nil {
77
+
return fmt.Errorf("create temp dir: %w", err)
78
+
}
79
+
defer os.RemoveAll(workdir)
80
+
81
+
return vendors.Sync(cmd.Context(), workdir, tunnel.LocalAddr, entries)
82
+
}
+113
toolbox/internal/vendors/config.go
+113
toolbox/internal/vendors/config.go
···
1
+
package vendors
2
+
3
+
import (
4
+
"fmt"
5
+
"os"
6
+
"sort"
7
+
"strings"
8
+
9
+
"gopkg.in/yaml.v3"
10
+
)
11
+
12
+
type Config struct {
13
+
Items map[string]Vendor `yaml:"vendors"`
14
+
}
15
+
16
+
type Vendor struct {
17
+
Kind string `yaml:"kind"`
18
+
RepoURL string `yaml:"repo_url,omitempty"`
19
+
Ref string `yaml:"ref,omitempty"`
20
+
Chart string `yaml:"chart,omitempty"`
21
+
Versions []string `yaml:"versions"`
22
+
Source string `yaml:"source,omitempty"`
23
+
}
24
+
25
+
type VendorEntry struct {
26
+
Name string
27
+
Vendor
28
+
}
29
+
30
+
func LoadConfig(configPath string) (*Config, error) {
31
+
data, err := os.ReadFile(configPath)
32
+
if err != nil {
33
+
return nil, fmt.Errorf("read file: %w", err)
34
+
}
35
+
36
+
var config Config
37
+
if err := yaml.Unmarshal(data, &config); err != nil {
38
+
return nil, fmt.Errorf("parse YAML: %w", err)
39
+
}
40
+
41
+
return &config, nil
42
+
}
43
+
44
+
func ParseAndValidate(config *Config) ([]VendorEntry, error) {
45
+
names := make([]string, 0, len(config.Items))
46
+
for name := range config.Items {
47
+
names = append(names, name)
48
+
}
49
+
sort.Strings(names)
50
+
51
+
entries := make([]VendorEntry, 0, len(names))
52
+
53
+
for _, name := range names {
54
+
vendor := config.Items[name]
55
+
56
+
if err := validateDestination(name); err != nil {
57
+
return nil, err
58
+
}
59
+
60
+
vendor.Kind = strings.ToLower(vendor.Kind)
61
+
62
+
if len(vendor.Versions) == 0 {
63
+
return nil, fmt.Errorf("vendors.%s: versions is required", name)
64
+
}
65
+
66
+
for _, version := range vendor.Versions {
67
+
if version == "" {
68
+
return nil, fmt.Errorf("vendors.%s: versions cannot be empty", name)
69
+
}
70
+
}
71
+
72
+
switch vendor.Kind {
73
+
case "chart":
74
+
if vendor.Ref != "" {
75
+
if vendor.RepoURL != "" || vendor.Chart != "" {
76
+
return nil, fmt.Errorf("vendors.%s: use either ref or repo_url/chart", name)
77
+
}
78
+
} else {
79
+
if vendor.RepoURL == "" || vendor.Chart == "" {
80
+
return nil, fmt.Errorf("vendors.%s: repo_url and chart are both required", name)
81
+
}
82
+
}
83
+
84
+
case "image":
85
+
if vendor.Source == "" {
86
+
return nil, fmt.Errorf("vendors.%s: source is required", name)
87
+
}
88
+
89
+
default:
90
+
if vendor.Kind == "" {
91
+
return nil, fmt.Errorf("vendors.%s: kind is required (chart|image)", name)
92
+
}
93
+
return nil, fmt.Errorf("vendors.%s: invalid kind %q", name, vendor.Kind)
94
+
}
95
+
96
+
entries = append(entries, VendorEntry{Name: name, Vendor: vendor})
97
+
}
98
+
99
+
return entries, nil
100
+
}
101
+
102
+
func validateDestination(destination string) error {
103
+
if destination == "" {
104
+
return fmt.Errorf("vendors: destination key is required")
105
+
}
106
+
if strings.Contains(destination, "://") {
107
+
return fmt.Errorf("vendors.%s: destination must be relative to the internal registry", destination)
108
+
}
109
+
if strings.HasPrefix(destination, "/") {
110
+
return fmt.Errorf("vendors.%s: destination must not start with /", destination)
111
+
}
112
+
return nil
113
+
}
+44
toolbox/internal/vendors/config_test.go
+44
toolbox/internal/vendors/config_test.go
···
1
+
package vendors
2
+
3
+
import (
4
+
"strings"
5
+
"testing"
6
+
)
7
+
8
+
func TestParseAndValidate(t *testing.T) {
9
+
cases := []struct {
10
+
name string
11
+
config *Config
12
+
wantErr string
13
+
}{
14
+
{"missing chart versions", &Config{Items: map[string]Vendor{
15
+
"vendor/charts/dex": {Kind: "chart", Chart: "dex", RepoURL: "https://charts.dexidp.io"},
16
+
}}, "versions is required"},
17
+
{"missing image kind", &Config{Items: map[string]Vendor{
18
+
"vendor/charts/dex": {Versions: []string{"0.23.0"}, Chart: "dex", RepoURL: "https://charts.dexidp.io"},
19
+
}}, "kind is required"},
20
+
{"image with source and no versions", &Config{Items: map[string]Vendor{
21
+
"vendor/images/dex": {Kind: "image", Source: "ghcr.io/dexidp/dex"},
22
+
}}, "versions is required"},
23
+
{"valid image versions", &Config{Items: map[string]Vendor{
24
+
"vendor/images/dex": {Kind: "image", Source: "ghcr.io/dexidp/dex", Versions: []string{"v2.43.1"}},
25
+
}}, ""},
26
+
}
27
+
28
+
for _, tc := range cases {
29
+
t.Run(tc.name, func(t *testing.T) {
30
+
_, err := ParseAndValidate(tc.config)
31
+
if tc.wantErr == "" {
32
+
if err != nil {
33
+
t.Fatalf("unexpected validation error: %v", err)
34
+
}
35
+
return
36
+
}
37
+
if err == nil {
38
+
t.Fatal("expected validation error, got nil")
39
+
} else if !strings.Contains(err.Error(), tc.wantErr) {
40
+
t.Fatalf("expected validation error %q, got %v", tc.wantErr, err)
41
+
}
42
+
})
43
+
}
44
+
}
+109
toolbox/internal/vendors/sync.go
+109
toolbox/internal/vendors/sync.go
···
1
+
package vendors
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"os"
7
+
"os/exec"
8
+
"path/filepath"
9
+
"strings"
10
+
11
+
"github.com/charmbracelet/log"
12
+
)
13
+
14
+
func LoadVendors(configPath string) ([]VendorEntry, error) {
15
+
config, err := LoadConfig(configPath)
16
+
if err != nil {
17
+
return nil, fmt.Errorf("load settings file: %w", err)
18
+
}
19
+
20
+
entries, err := ParseAndValidate(config)
21
+
if err != nil {
22
+
return nil, fmt.Errorf("validate settings: %w", err)
23
+
}
24
+
25
+
return entries, nil
26
+
}
27
+
28
+
func Sync(ctx context.Context, workdir, registryAddr string, entries []VendorEntry) error {
29
+
for _, item := range entries {
30
+
switch item.Kind {
31
+
case "chart":
32
+
if err := syncChart(ctx, workdir, registryAddr, item); err != nil {
33
+
return err
34
+
}
35
+
case "image":
36
+
if err := syncImage(ctx, registryAddr, item); err != nil {
37
+
return err
38
+
}
39
+
}
40
+
}
41
+
return nil
42
+
}
43
+
44
+
func syncChart(ctx context.Context, workdir, registryAddr string, chart VendorEntry) error {
45
+
chartDir := filepath.Join(workdir, chart.Name)
46
+
if err := os.MkdirAll(chartDir, 0o755); err != nil {
47
+
return fmt.Errorf("create chart temp dir: %w", err)
48
+
}
49
+
50
+
pullRef := chart.Chart
51
+
if chart.Ref != "" {
52
+
pullRef = chart.Ref
53
+
}
54
+
for _, version := range chart.Versions {
55
+
log.Infof("vendoring chart %s@%s", chart.Name, version)
56
+
57
+
pullArgs := []string{"pull", pullRef, "--version", version, "--destination", chartDir}
58
+
if chart.RepoURL != "" {
59
+
pullArgs = append(pullArgs, "--repo", chart.RepoURL)
60
+
}
61
+
if err := runCommand(ctx, "helm", pullArgs...); err != nil {
62
+
return fmt.Errorf("pull chart %s@%s: %w", chart.Name, version, err)
63
+
}
64
+
65
+
archivePath := filepath.Join(chartDir, filepath.Base(pullRef)+"-"+version+".tgz")
66
+
pushTarget := fmt.Sprintf("oci://%s/%s", registryAddr, chart.Name)
67
+
if err := runCommand(ctx, "helm", "push", archivePath, pushTarget, "--plain-http"); err != nil {
68
+
return fmt.Errorf("push chart %s@%s: %w", chart.Name, version, err)
69
+
}
70
+
}
71
+
72
+
return nil
73
+
}
74
+
75
+
func syncImage(ctx context.Context, registryAddr string, image VendorEntry) error {
76
+
for _, version := range image.Versions {
77
+
log.Infof("vendoring image %s:%s", image.Name, version)
78
+
79
+
source := image.Source
80
+
target := image.Name
81
+
if strings.HasPrefix(version, "@") {
82
+
source += version
83
+
target += version
84
+
} else {
85
+
source += ":" + version
86
+
target += ":" + version
87
+
}
88
+
destination := fmt.Sprintf("%s/%s", registryAddr, target)
89
+
copyArgs := []string{"cp", source, destination, "--to-plain-http"}
90
+
91
+
if err := runCommand(ctx, "oras", copyArgs...); err != nil {
92
+
return fmt.Errorf("copy image %s@%s: %w", image.Name, version, err)
93
+
}
94
+
}
95
+
96
+
return nil
97
+
}
98
+
99
+
func runCommand(ctx context.Context, name string, args ...string) error {
100
+
cmd := exec.CommandContext(ctx, name, args...)
101
+
cmd.Stdout = os.Stdout
102
+
cmd.Stderr = os.Stderr
103
+
104
+
if err := cmd.Run(); err != nil {
105
+
return fmt.Errorf("%s %v: %w", name, args, err)
106
+
}
107
+
108
+
return nil
109
+
}