knotserver, knotmirror: fix language detection accuracy and timeouts #307

Increases language analysis HTTP timeouts from 1s to 5s (and 10s for post-receive hooks) to prevent context cancellation on larger repositories.
Implements a 10,000 file cap in AnalyzeLanguages to prevent I/O starvation on massive monorepos.
Excludes enry.Unknown files from language calculations to prevent huge asset files (like Godot .tscn/.tres) from completely washing out the true code languages.

Signed-off-by: atixnotfound.tngl.sh atsharma623@gmail.com

+17 -5

4 changed files

Diff round #0

expand all

knotmirror

xrpc

git_list_languages.go

knotserver

git

language.go

post_receive.go

xrpc

repo_languages.go

+1 -1

knotmirror/xrpc/git_list_languages.go

··· 51 51 return nil, &atclient.APIError{StatusCode: http.StatusNotFound, Name: "RepoNotFound", Message: "failed to find git repo"} 52 52 } 53 53 54 - ctx, cancel := context.WithTimeout(ctx, 1*time.Second) 54 + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) 55 55 defer cancel() 56 56 57 57 sizes, err := gr.AnalyzeLanguages(ctx)

+14 -2

knotserver/git/language.go

··· 11 11 12 12 type LangBreakdown map[string]int64 13 13 14 + // maxAnalyzeFiles caps how many files are inspected to avoid runaway walks on 15 + // extremely large repositories. 16 + const maxAnalyzeFiles = 10_000 17 + 14 18 func (g *GitRepo) AnalyzeLanguages(ctx context.Context) (LangBreakdown, error) { 15 19 sizes := make(map[string]int64) 20 + fileCount := 0 21 + 16 22 err := g.Walk(ctx, "", func(node object.TreeEntry, parent *object.Tree, root string) error { 23 + if fileCount >= maxAnalyzeFiles { 24 + return TerminateWalk 25 + } 26 + fileCount++ 27 + 17 28 filepath := path.Join(root, node.Name) 18 29 19 - content, err := g.FileContentN(filepath, 16*1024) // 16KB 30 + content, err := g.FileContentN(filepath, 16*1024) // 16 KB 20 31 if err != nil { 21 32 return nil 22 33 } ··· 33 44 } 34 45 35 46 langType := enry.GetLanguageType(language) 36 - if langType != enry.Programming && langType != enry.Markup && langType != enry.Unknown { 47 + if langType != enry.Programming && langType != enry.Markup { 37 48 return nil 38 49 } 39 50 ··· 67 78 68 79 return enry.GetLanguage(node.Name, content) 69 80 } 81 +

+1 -1

knotserver/git/post_receive.go

··· 67 67 isDefaultRef, err := g.isDefaultBranch(line) 68 68 errors.Join(errs, err) 69 69 70 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) 70 + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 71 71 defer cancel() 72 72 breakdown, err := g.AnalyzeLanguages(ctx) 73 73 errors.Join(errs, err)

+1 -1

knotserver/xrpc/repo_languages.go

··· 28 28 return 29 29 } 30 30 31 - ctx, cancel := context.WithTimeout(r.Context(), 1*time.Second) 31 + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) 32 32 defer cancel() 33 33 34 34 sizes, err := gr.AnalyzeLanguages(ctx)

History

1 round 0 comments

atixnotfound.tngl.sh submitted #0 2w

1 commit

expand

953c97a1

knotserver, knotmirror: fix language detection accuracy and timeouts

merge conflicts detected

expand

knotmirror/xrpc/git_list_languages.go:51
knotserver/git/language.go:11

expand 0 comments

Configure Feed

Configure Feed

History