this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat(toolbox): vendor artifacts into internal registry

+349
+1
toolbox/cmd/root.go
··· 28 28 29 29 rootCmd.AddCommand(gitopsCmd) 30 30 rootCmd.AddCommand(secretsCmd) 31 + rootCmd.AddCommand(vendorCmd) 31 32 } 32 33 33 34 var rootCmd = &cobra.Command{
+82
toolbox/cmd/vendor.go
··· 1 + package cmd 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "os" 7 + "os/exec" 8 + 9 + "github.com/spf13/cobra" 10 + 11 + "github.com/khuedoan/cloudlab/toolbox/internal/cluster" 12 + "github.com/khuedoan/cloudlab/toolbox/internal/vendors" 13 + ) 14 + 15 + var vendorCmd = &cobra.Command{ 16 + Use: "vendor", 17 + Args: cobra.NoArgs, 18 + Short: "Vendor charts and images from settings.yaml into the in-cluster registry", 19 + PreRunE: func(_ *cobra.Command, _ []string) error { 20 + if err := validateClusterFlags(); err != nil { 21 + return err 22 + } 23 + if settingsFile == "" { 24 + return fmt.Errorf("--settings is required") 25 + } 26 + for _, name := range []string{"helm", "oras"} { 27 + if _, err := exec.LookPath(name); err != nil { 28 + return fmt.Errorf("find %s CLI: %w", name, err) 29 + } 30 + } 31 + return nil 32 + }, 33 + RunE: runSync, 34 + } 35 + 36 + func init() { 37 + vendorCmd.Flags().StringVar(&settingsFile, "settings", "", "Path to settings YAML file") 38 + } 39 + 40 + func runSync(cmd *cobra.Command, _ []string) error { 41 + entries, err := vendors.LoadVendors(settingsFile) 42 + if err != nil { 43 + return err 44 + } 45 + 46 + connectCtx, cancel := context.WithTimeout(cmd.Context(), connectTimeout) 47 + defer cancel() 48 + 49 + hostAddr, err := cluster.LoadHost(hostsFile, host) 50 + if err != nil { 51 + return fmt.Errorf("load host: %w", err) 52 + } 53 + 54 + conn, err := cluster.Connect(cluster.SSHConfig{ 55 + Host: hostAddr, 56 + User: sshUser, 57 + KeyPath: sshKey, 58 + KnownHostsPath: sshKnownHosts, 59 + Timeout: connectTimeout, 60 + }) 61 + if err != nil { 62 + return fmt.Errorf("connect to cluster: %w", err) 63 + } 64 + defer conn.Close() 65 + 66 + tunnel, err := conn.Forward(connectCtx, cluster.ServiceConfig{ 67 + Namespace: registryNamespace, 68 + Name: registryService, 69 + Port: registryPort, 70 + }) 71 + if err != nil { 72 + return fmt.Errorf("forward registry: %w", err) 73 + } 74 + 75 + workdir, err := os.MkdirTemp("", "toolbox-vendor-*") 76 + if err != nil { 77 + return fmt.Errorf("create temp dir: %w", err) 78 + } 79 + defer os.RemoveAll(workdir) 80 + 81 + return vendors.Sync(cmd.Context(), workdir, tunnel.LocalAddr, entries) 82 + }
+113
toolbox/internal/vendors/config.go
··· 1 + package vendors 2 + 3 + import ( 4 + "fmt" 5 + "os" 6 + "sort" 7 + "strings" 8 + 9 + "gopkg.in/yaml.v3" 10 + ) 11 + 12 + type Config struct { 13 + Items map[string]Vendor `yaml:"vendors"` 14 + } 15 + 16 + type Vendor struct { 17 + Kind string `yaml:"kind"` 18 + RepoURL string `yaml:"repo_url,omitempty"` 19 + Ref string `yaml:"ref,omitempty"` 20 + Chart string `yaml:"chart,omitempty"` 21 + Versions []string `yaml:"versions"` 22 + Source string `yaml:"source,omitempty"` 23 + } 24 + 25 + type VendorEntry struct { 26 + Name string 27 + Vendor 28 + } 29 + 30 + func LoadConfig(configPath string) (*Config, error) { 31 + data, err := os.ReadFile(configPath) 32 + if err != nil { 33 + return nil, fmt.Errorf("read file: %w", err) 34 + } 35 + 36 + var config Config 37 + if err := yaml.Unmarshal(data, &config); err != nil { 38 + return nil, fmt.Errorf("parse YAML: %w", err) 39 + } 40 + 41 + return &config, nil 42 + } 43 + 44 + func ParseAndValidate(config *Config) ([]VendorEntry, error) { 45 + names := make([]string, 0, len(config.Items)) 46 + for name := range config.Items { 47 + names = append(names, name) 48 + } 49 + sort.Strings(names) 50 + 51 + entries := make([]VendorEntry, 0, len(names)) 52 + 53 + for _, name := range names { 54 + vendor := config.Items[name] 55 + 56 + if err := validateDestination(name); err != nil { 57 + return nil, err 58 + } 59 + 60 + vendor.Kind = strings.ToLower(vendor.Kind) 61 + 62 + if len(vendor.Versions) == 0 { 63 + return nil, fmt.Errorf("vendors.%s: versions is required", name) 64 + } 65 + 66 + for _, version := range vendor.Versions { 67 + if version == "" { 68 + return nil, fmt.Errorf("vendors.%s: versions cannot be empty", name) 69 + } 70 + } 71 + 72 + switch vendor.Kind { 73 + case "chart": 74 + if vendor.Ref != "" { 75 + if vendor.RepoURL != "" || vendor.Chart != "" { 76 + return nil, fmt.Errorf("vendors.%s: use either ref or repo_url/chart", name) 77 + } 78 + } else { 79 + if vendor.RepoURL == "" || vendor.Chart == "" { 80 + return nil, fmt.Errorf("vendors.%s: repo_url and chart are both required", name) 81 + } 82 + } 83 + 84 + case "image": 85 + if vendor.Source == "" { 86 + return nil, fmt.Errorf("vendors.%s: source is required", name) 87 + } 88 + 89 + default: 90 + if vendor.Kind == "" { 91 + return nil, fmt.Errorf("vendors.%s: kind is required (chart|image)", name) 92 + } 93 + return nil, fmt.Errorf("vendors.%s: invalid kind %q", name, vendor.Kind) 94 + } 95 + 96 + entries = append(entries, VendorEntry{Name: name, Vendor: vendor}) 97 + } 98 + 99 + return entries, nil 100 + } 101 + 102 + func validateDestination(destination string) error { 103 + if destination == "" { 104 + return fmt.Errorf("vendors: destination key is required") 105 + } 106 + if strings.Contains(destination, "://") { 107 + return fmt.Errorf("vendors.%s: destination must be relative to the internal registry", destination) 108 + } 109 + if strings.HasPrefix(destination, "/") { 110 + return fmt.Errorf("vendors.%s: destination must not start with /", destination) 111 + } 112 + return nil 113 + }
+44
toolbox/internal/vendors/config_test.go
··· 1 + package vendors 2 + 3 + import ( 4 + "strings" 5 + "testing" 6 + ) 7 + 8 + func TestParseAndValidate(t *testing.T) { 9 + cases := []struct { 10 + name string 11 + config *Config 12 + wantErr string 13 + }{ 14 + {"missing chart versions", &Config{Items: map[string]Vendor{ 15 + "vendor/charts/dex": {Kind: "chart", Chart: "dex", RepoURL: "https://charts.dexidp.io"}, 16 + }}, "versions is required"}, 17 + {"missing image kind", &Config{Items: map[string]Vendor{ 18 + "vendor/charts/dex": {Versions: []string{"0.23.0"}, Chart: "dex", RepoURL: "https://charts.dexidp.io"}, 19 + }}, "kind is required"}, 20 + {"image with source and no versions", &Config{Items: map[string]Vendor{ 21 + "vendor/images/dex": {Kind: "image", Source: "ghcr.io/dexidp/dex"}, 22 + }}, "versions is required"}, 23 + {"valid image versions", &Config{Items: map[string]Vendor{ 24 + "vendor/images/dex": {Kind: "image", Source: "ghcr.io/dexidp/dex", Versions: []string{"v2.43.1"}}, 25 + }}, ""}, 26 + } 27 + 28 + for _, tc := range cases { 29 + t.Run(tc.name, func(t *testing.T) { 30 + _, err := ParseAndValidate(tc.config) 31 + if tc.wantErr == "" { 32 + if err != nil { 33 + t.Fatalf("unexpected validation error: %v", err) 34 + } 35 + return 36 + } 37 + if err == nil { 38 + t.Fatal("expected validation error, got nil") 39 + } else if !strings.Contains(err.Error(), tc.wantErr) { 40 + t.Fatalf("expected validation error %q, got %v", tc.wantErr, err) 41 + } 42 + }) 43 + } 44 + }
+109
toolbox/internal/vendors/sync.go
··· 1 + package vendors 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "os" 7 + "os/exec" 8 + "path/filepath" 9 + "strings" 10 + 11 + "github.com/charmbracelet/log" 12 + ) 13 + 14 + func LoadVendors(configPath string) ([]VendorEntry, error) { 15 + config, err := LoadConfig(configPath) 16 + if err != nil { 17 + return nil, fmt.Errorf("load settings file: %w", err) 18 + } 19 + 20 + entries, err := ParseAndValidate(config) 21 + if err != nil { 22 + return nil, fmt.Errorf("validate settings: %w", err) 23 + } 24 + 25 + return entries, nil 26 + } 27 + 28 + func Sync(ctx context.Context, workdir, registryAddr string, entries []VendorEntry) error { 29 + for _, item := range entries { 30 + switch item.Kind { 31 + case "chart": 32 + if err := syncChart(ctx, workdir, registryAddr, item); err != nil { 33 + return err 34 + } 35 + case "image": 36 + if err := syncImage(ctx, registryAddr, item); err != nil { 37 + return err 38 + } 39 + } 40 + } 41 + return nil 42 + } 43 + 44 + func syncChart(ctx context.Context, workdir, registryAddr string, chart VendorEntry) error { 45 + chartDir := filepath.Join(workdir, chart.Name) 46 + if err := os.MkdirAll(chartDir, 0o755); err != nil { 47 + return fmt.Errorf("create chart temp dir: %w", err) 48 + } 49 + 50 + pullRef := chart.Chart 51 + if chart.Ref != "" { 52 + pullRef = chart.Ref 53 + } 54 + for _, version := range chart.Versions { 55 + log.Infof("vendoring chart %s@%s", chart.Name, version) 56 + 57 + pullArgs := []string{"pull", pullRef, "--version", version, "--destination", chartDir} 58 + if chart.RepoURL != "" { 59 + pullArgs = append(pullArgs, "--repo", chart.RepoURL) 60 + } 61 + if err := runCommand(ctx, "helm", pullArgs...); err != nil { 62 + return fmt.Errorf("pull chart %s@%s: %w", chart.Name, version, err) 63 + } 64 + 65 + archivePath := filepath.Join(chartDir, filepath.Base(pullRef)+"-"+version+".tgz") 66 + pushTarget := fmt.Sprintf("oci://%s/%s", registryAddr, chart.Name) 67 + if err := runCommand(ctx, "helm", "push", archivePath, pushTarget, "--plain-http"); err != nil { 68 + return fmt.Errorf("push chart %s@%s: %w", chart.Name, version, err) 69 + } 70 + } 71 + 72 + return nil 73 + } 74 + 75 + func syncImage(ctx context.Context, registryAddr string, image VendorEntry) error { 76 + for _, version := range image.Versions { 77 + log.Infof("vendoring image %s:%s", image.Name, version) 78 + 79 + source := image.Source 80 + target := image.Name 81 + if strings.HasPrefix(version, "@") { 82 + source += version 83 + target += version 84 + } else { 85 + source += ":" + version 86 + target += ":" + version 87 + } 88 + destination := fmt.Sprintf("%s/%s", registryAddr, target) 89 + copyArgs := []string{"cp", source, destination, "--to-plain-http"} 90 + 91 + if err := runCommand(ctx, "oras", copyArgs...); err != nil { 92 + return fmt.Errorf("copy image %s@%s: %w", image.Name, version, err) 93 + } 94 + } 95 + 96 + return nil 97 + } 98 + 99 + func runCommand(ctx context.Context, name string, args ...string) error { 100 + cmd := exec.CommandContext(ctx, name, args...) 101 + cmd.Stdout = os.Stdout 102 + cmd.Stderr = os.Stderr 103 + 104 + if err := cmd.Run(); err != nil { 105 + return fmt.Errorf("%s %v: %w", name, args, err) 106 + } 107 + 108 + return nil 109 + }