this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat: detect changed modules and strict graph type

+283 -32
+63 -1
controller/activities/git.go
··· 3 3 import ( 4 4 "context" 5 5 "os" 6 + "path/filepath" 7 + "strings" 6 8 7 9 "github.com/go-git/go-git/v5" 8 10 "github.com/go-git/go-git/v5/plumbing" ··· 29 31 return path, nil 30 32 } 31 33 32 - func ChangedFiles(ctx context.Context, path string, oldRevision string) ([]string, error) { 34 + func changedFiles(ctx context.Context, path string, oldRevision string) ([]string, error) { 33 35 logger := activity.GetLogger(ctx) 34 36 logger.Info("Getting changed files", "path", path, "oldRevision", oldRevision) 35 37 ··· 93 95 94 96 return files, nil 95 97 } 98 + 99 + func ChangedModules(ctx context.Context, repoPath string, oldRevision string) ([]string, error) { 100 + logger := activity.GetLogger(ctx) 101 + logger.Info("Getting changed modules", "path", repoPath, "oldRevision", oldRevision) 102 + 103 + // Get all changed files 104 + changedFiles, err := changedFiles(ctx, repoPath, oldRevision) 105 + if err != nil { 106 + return nil, err 107 + } 108 + 109 + seen := make(map[string]struct{}) 110 + var modules []string 111 + 112 + for _, file := range changedFiles { 113 + // Get the directory of the changed file 114 + dir := filepath.Dir(file) 115 + 116 + // Walk up the directory tree to find the closest directory containing terragrunt.hcl 117 + currentDir := dir 118 + for { 119 + terragruntPath := filepath.Join(repoPath, currentDir, "terragrunt.hcl") 120 + if _, err := os.Stat(terragruntPath); err == nil { 121 + // Found terragrunt.hcl, this is a module directory 122 + modulePath := currentDir 123 + 124 + // Remove infra/<env>/ prefix if present 125 + if strings.HasPrefix(modulePath, "infra/") { 126 + parts := strings.Split(filepath.ToSlash(modulePath), "/") 127 + if len(parts) >= 3 && parts[0] == "infra" { 128 + // Remove "infra" and environment (e.g., "dev", "prod") 129 + modulePath = strings.Join(parts[2:], "/") 130 + } 131 + } 132 + 133 + // Skip empty paths 134 + if modulePath != "" && modulePath != "." { 135 + // Normalize path separators to forward slashes 136 + modulePath = filepath.ToSlash(modulePath) 137 + 138 + if _, exists := seen[modulePath]; !exists { 139 + modules = append(modules, modulePath) 140 + seen[modulePath] = struct{}{} 141 + } 142 + } 143 + break 144 + } 145 + 146 + // Move up one directory level 147 + parent := filepath.Dir(currentDir) 148 + if parent == currentDir || parent == "." { 149 + // Reached the root, no terragrunt.hcl found 150 + break 151 + } 152 + currentDir = parent 153 + } 154 + } 155 + 156 + return modules, nil 157 + }
+183
controller/activities/git_test.go
··· 1 + package activities 2 + 3 + import ( 4 + "context" 5 + "os" 6 + "path/filepath" 7 + "reflect" 8 + "sort" 9 + "strings" 10 + "testing" 11 + ) 12 + 13 + func TestChangedModules(t *testing.T) { 14 + // Create a temporary directory structure for testing 15 + tempDir, err := os.MkdirTemp("", "test-git-") 16 + if err != nil { 17 + t.Fatalf("Failed to create temp dir: %v", err) 18 + } 19 + defer os.RemoveAll(tempDir) 20 + 21 + // Create test directory structure 22 + testDirs := []string{ 23 + "infra/dev/core", 24 + "infra/dev/networking", 25 + "infra/dev/databases/postgres", 26 + "infra/prod/core", 27 + "infra/prod/monitoring", 28 + "shared/modules/vpc", 29 + "shared/modules/security", 30 + "docs", 31 + } 32 + 33 + for _, dir := range testDirs { 34 + err := os.MkdirAll(filepath.Join(tempDir, dir), 0755) 35 + if err != nil { 36 + t.Fatalf("Failed to create dir %s: %v", dir, err) 37 + } 38 + } 39 + 40 + // Create terragrunt.hcl files in specific directories 41 + terragruntDirs := []string{ 42 + "infra/dev/core", 43 + "infra/dev/networking", 44 + "infra/dev/databases/postgres", 45 + "infra/prod/core", 46 + "infra/prod/monitoring", 47 + "shared/modules/vpc", 48 + } 49 + 50 + for _, dir := range terragruntDirs { 51 + terragruntPath := filepath.Join(tempDir, dir, "terragrunt.hcl") 52 + err := os.WriteFile(terragruntPath, []byte("# terragrunt config"), 0644) 53 + if err != nil { 54 + t.Fatalf("Failed to create terragrunt.hcl in %s: %v", dir, err) 55 + } 56 + } 57 + 58 + // Test cases with mock changed files 59 + testCases := []struct { 60 + name string 61 + changedFiles []string 62 + expected []string 63 + }{ 64 + { 65 + name: "Single module change", 66 + changedFiles: []string{ 67 + "infra/dev/core/main.tf", 68 + "infra/dev/core/variables.tf", 69 + }, 70 + expected: []string{"core"}, 71 + }, 72 + { 73 + name: "Multiple modules in same environment", 74 + changedFiles: []string{ 75 + "infra/dev/core/main.tf", 76 + "infra/dev/networking/vpc.tf", 77 + "infra/dev/databases/postgres/db.tf", 78 + }, 79 + expected: []string{"core", "networking", "databases/postgres"}, 80 + }, 81 + { 82 + name: "Modules across different environments", 83 + changedFiles: []string{ 84 + "infra/dev/core/main.tf", 85 + "infra/prod/core/main.tf", 86 + "infra/prod/monitoring/alerts.tf", 87 + }, 88 + expected: []string{"core", "monitoring"}, 89 + }, 90 + { 91 + name: "Shared modules without infra prefix", 92 + changedFiles: []string{ 93 + "shared/modules/vpc/main.tf", 94 + "shared/modules/vpc/outputs.tf", 95 + }, 96 + expected: []string{"shared/modules/vpc"}, 97 + }, 98 + { 99 + name: "Files without terragrunt.hcl", 100 + changedFiles: []string{ 101 + "docs/README.md", 102 + "shared/modules/security/policy.tf", // no terragrunt.hcl in security dir 103 + }, 104 + expected: []string{}, 105 + }, 106 + { 107 + name: "Mixed files with and without terragrunt.hcl", 108 + changedFiles: []string{ 109 + "infra/dev/core/main.tf", 110 + "docs/README.md", 111 + "shared/modules/vpc/vpc.tf", 112 + }, 113 + expected: []string{"core", "shared/modules/vpc"}, 114 + }, 115 + } 116 + 117 + for _, tc := range testCases { 118 + t.Run(tc.name, func(t *testing.T) { 119 + // Create a mock ChangedModules function that uses our test data 120 + // We'll create a custom function that simulates the file system checks 121 + modules := getChangedModulesFromFiles(tempDir, tc.changedFiles) 122 + 123 + sort.Strings(modules) 124 + sort.Strings(tc.expected) 125 + 126 + if !reflect.DeepEqual(modules, tc.expected) { 127 + t.Errorf("Expected modules %v, but got %v", tc.expected, modules) 128 + } 129 + }) 130 + } 131 + } 132 + 133 + // Helper function to simulate ChangedModules logic without Git 134 + func getChangedModulesFromFiles(repoPath string, changedFiles []string) []string { 135 + seen := make(map[string]struct{}) 136 + var modules []string 137 + 138 + for _, file := range changedFiles { 139 + // Get the directory of the changed file 140 + dir := filepath.Dir(file) 141 + 142 + // Walk up the directory tree to find the closest directory containing terragrunt.hcl 143 + currentDir := dir 144 + for { 145 + terragruntPath := filepath.Join(repoPath, currentDir, "terragrunt.hcl") 146 + if _, err := os.Stat(terragruntPath); err == nil { 147 + // Found terragrunt.hcl, this is a module directory 148 + modulePath := currentDir 149 + 150 + // Remove infra/<env>/ prefix if present 151 + if len(modulePath) > 0 && filepath.HasPrefix(modulePath, "infra/") { 152 + parts := strings.Split(filepath.ToSlash(modulePath), "/") 153 + if len(parts) >= 3 && parts[0] == "infra" { 154 + // Remove "infra" and environment (e.g., "dev", "prod") 155 + modulePath = strings.Join(parts[2:], "/") 156 + } 157 + } 158 + 159 + // Skip empty paths 160 + if modulePath != "" && modulePath != "." { 161 + // Normalize path separators to forward slashes 162 + modulePath = filepath.ToSlash(modulePath) 163 + 164 + if _, exists := seen[modulePath]; !exists { 165 + modules = append(modules, modulePath) 166 + seen[modulePath] = struct{}{} 167 + } 168 + } 169 + break 170 + } 171 + 172 + // Move up one directory level 173 + parent := filepath.Dir(currentDir) 174 + if parent == currentDir || parent == "." { 175 + // Reached the root, no terragrunt.hcl found 176 + break 177 + } 178 + currentDir = parent 179 + } 180 + } 181 + 182 + return modules 183 + }
-1
controller/activities/graph_types.go
··· 1 - package activities
+13 -13
controller/activities/terragrunt.go
··· 8 8 "go.temporal.io/sdk/activity" 9 9 ) 10 10 11 - func TerragruntGraph(ctx context.Context, path string) (string, error) { 11 + func TerragruntGraph(ctx context.Context, path string) (*Graph, error) { 12 12 logger := activity.GetLogger(ctx) 13 13 logger.Info("Generating Terragrunt DAG graph", "path", path) 14 14 ··· 16 16 cmd.Dir = path 17 17 output, err := cmd.Output() 18 18 if err != nil { 19 - return "", fmt.Errorf("failed to run terragrunt dag graph: %w", err) 19 + return nil, fmt.Errorf("failed to run terragrunt dag graph: %w", err) 20 + } 21 + 22 + graph, err := NewGraphFromDot(string(output)) 23 + if err != nil { 24 + return nil, fmt.Errorf("failed to parse terragrunt graph output: %w", err) 20 25 } 21 26 22 - return string(output), nil 27 + return graph, nil 23 28 } 24 29 25 - func TerragruntGraphShaking(ctx context.Context, dotGraph string, changedFiles []string) (string, error) { 30 + func TerragruntGraphShaking(ctx context.Context, graph *Graph, changedModules []string) (*Graph, error) { 26 31 logger := activity.GetLogger(ctx) 27 32 28 - logger.Info("Parsing Terragrunt DAG graph") 33 + logger.Info("Pruning Terragrunt DAG graph") 29 34 30 - graph, err := NewGraphFromDot(dotGraph) 35 + prunedGraph, err := PruneGraph(ctx, graph, changedModules) 31 36 if err != nil { 32 - return "", fmt.Errorf("failed to parse dot graph: %w", err) 37 + return nil, fmt.Errorf("failed to prune dependency graph: %w", err) 33 38 } 34 39 35 - prunedGraph, err := PruneGraph(ctx, graph, changedFiles) 36 - if err != nil { 37 - return "", fmt.Errorf("failed to prune dependency graph: %w", err) 38 - } 39 - 40 - return prunedGraph.ToDot(), nil 40 + return prunedGraph, nil 41 41 }
+1 -1
controller/worker/main.go
··· 24 24 25 25 w.RegisterWorkflow(workflows.Infra) 26 26 w.RegisterActivity(activities.Clone) 27 - w.RegisterActivity(activities.ChangedFiles) 27 + w.RegisterActivity(activities.ChangedModules) 28 28 w.RegisterActivity(activities.TerragruntGraph) 29 29 w.RegisterActivity(activities.TerragruntGraphShaking) 30 30
+23 -16
controller/workflows/infra.go
··· 18 18 // For now do that manually on the UI 19 19 // Task queue: cloudlab 20 20 // Workflow: Infra 21 - // Input json/plain: {"url": "https://github.com/khuedoan/cloudlab", "revision": "infra-rewrite", "stack": "local"} 22 - func Infra(ctx workflow.Context, input InfraInputs) (string, error) { 21 + // Input json/plain: 22 + // 23 + // { 24 + // "url": "https://github.com/khuedoan/cloudlab", 25 + // "revision": "infra-rewrite", 26 + // "oldRevision": "7796870a3c17105d7a13c5b6c990fa895de64952", 27 + // "stack": "local" 28 + // } 29 + func Infra(ctx workflow.Context, input InfraInputs) (*activities.Graph, error) { 23 30 ao := workflow.ActivityOptions{ 24 31 StartToCloseTimeout: 10 * time.Second, 25 32 } ··· 32 39 err := workflow.ExecuteActivity(ctx, activities.Clone, input.Url, input.Revision).Get(ctx, &path) 33 40 if err != nil { 34 41 logger.Error("Activity failed.", "Error", err) 35 - return "", err 42 + return nil, err 36 43 } 37 44 38 45 var ( 39 - dotGraph string 40 - changedFiles []string 46 + graph *activities.Graph 47 + changedModules []string 41 48 ) 42 49 43 50 graphFuture := workflow.ExecuteActivity(ctx, activities.TerragruntGraph, path+"/infra/"+input.Stack) 44 - changedFilesFuture := workflow.ExecuteActivity(ctx, activities.ChangedFiles, path, input.OldRevision) 51 + changedModulesFuture := workflow.ExecuteActivity(ctx, activities.ChangedModules, path, input.OldRevision) 45 52 46 - err = graphFuture.Get(ctx, &dotGraph) 53 + err = graphFuture.Get(ctx, &graph) 47 54 if err != nil { 48 55 logger.Error("TerragruntGraph failed", "Error", err) 49 - return "", err 56 + return nil, err 50 57 } 51 58 52 - err = changedFilesFuture.Get(ctx, &changedFiles) 59 + err = changedModulesFuture.Get(ctx, &changedModules) 53 60 if err != nil { 54 - logger.Error("ChangedFiles failed", "Error", err) 55 - return "", err 61 + logger.Error("ChangedModules failed", "Error", err) 62 + return nil, err 56 63 } 57 64 58 - var result string 59 - err = workflow.ExecuteActivity(ctx, activities.TerragruntGraphShaking, dotGraph, changedFiles).Get(ctx, &result) 65 + var prunedGraph *activities.Graph 66 + err = workflow.ExecuteActivity(ctx, activities.TerragruntGraphShaking, graph, changedModules).Get(ctx, &prunedGraph) 60 67 if err != nil { 61 68 logger.Error("Activity failed.", "Error", err) 62 - return "", err 69 + return nil, err 63 70 } 64 71 65 - logger.Info("Infra workflow completed.", "result", result) 72 + logger.Info("Infra workflow completed.", "nodes", len(prunedGraph.Nodes), "edges", len(prunedGraph.Edges)) 66 73 67 - return result, nil 74 + return prunedGraph, nil 68 75 }