loading up the forgejo repo on tangled to test page performance
Change use of Walk to WalkDir to improve disk performance (#22462)
As suggest by Go developers, use `filepath.WalkDir` instead of
`filepath.Walk` because [*Walk is less efficient than WalkDir,
introduced in Go 1.16, which avoids calling `os.Lstat` on every file or
directory visited](https://pkg.go.dev/path/filepath#Walk).
This proposition address that, in a similar way as
https://github.com/go-gitea/gitea/pull/22392 did.
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
authored by
Felipe Leopoldo Sologuren Gutiérrez
zeripath
Lunny Xiao
and committed by
GitHub
04c97aa3
da274380
+32
-18
+6
-2
build/generate-bindata.go
+6
-2
build/generate-bindata.go
···
32
32
33
33
hasher := sha1.New()
34
34
35
-
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
35
+
err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
36
36
if err != nil {
37
37
return err
38
38
}
39
-
_, _ = hasher.Write([]byte(info.Name()))
39
+
info, err := d.Info()
40
+
if err != nil {
41
+
return err
42
+
}
43
+
_, _ = hasher.Write([]byte(d.Name()))
40
44
_, _ = hasher.Write([]byte(info.ModTime().String()))
41
45
_, _ = hasher.Write([]byte(strconv.FormatInt(info.Size(), 16)))
42
46
return nil
+12
-2
modules/log/file.go
+12
-2
modules/log/file.go
···
225
225
226
226
func (log *FileLogger) deleteOldLog() {
227
227
dir := filepath.Dir(log.Filename)
228
-
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) (returnErr error) {
228
+
_ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) (returnErr error) {
229
229
defer func() {
230
230
if r := recover(); r != nil {
231
231
returnErr = fmt.Errorf("Unable to delete old log '%s', error: %+v", path, r)
232
232
}
233
233
}()
234
234
235
-
if !info.IsDir() && info.ModTime().Unix() < (time.Now().Unix()-60*60*24*log.Maxdays) {
235
+
if err != nil {
236
+
return err
237
+
}
238
+
if d.IsDir() {
239
+
return nil
240
+
}
241
+
info, err := d.Info()
242
+
if err != nil {
243
+
return err
244
+
}
245
+
if info.ModTime().Unix() < (time.Now().Unix() - 60*60*24*log.Maxdays) {
236
246
if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) {
237
247
if err := util.Remove(path); err != nil {
238
248
returnErr = fmt.Errorf("Failed to remove %s: %w", path, err)
+2
-2
modules/repository/generate.go
+2
-2
modules/repository/generate.go
···
173
173
// Avoid walking tree if there are no globs
174
174
if len(gt.Globs()) > 0 {
175
175
tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/"
176
-
if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error {
176
+
if err := filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error {
177
177
if walkErr != nil {
178
178
return walkErr
179
179
}
180
180
181
-
if info.IsDir() {
181
+
if d.IsDir() {
182
182
return nil
183
183
}
184
184
+2
-2
modules/storage/local.go
+2
-2
modules/storage/local.go
···
129
129
130
130
// IterateObjects iterates across the objects in the local storage
131
131
func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error {
132
-
return filepath.Walk(l.dir, func(path string, info os.FileInfo, err error) error {
132
+
return filepath.WalkDir(l.dir, func(path string, d os.DirEntry, err error) error {
133
133
if err != nil {
134
134
return err
135
135
}
···
141
141
if path == l.dir {
142
142
return nil
143
143
}
144
-
if info.IsDir() {
144
+
if d.IsDir() {
145
145
return nil
146
146
}
147
147
relPath, err := filepath.Rel(l.dir, path)
+4
-4
routers/web/user/setting/profile.go
+4
-4
routers/web/user/setting/profile.go
···
280
280
repos := map[string]*repo_model.Repository{}
281
281
// We're going to iterate by pagesize.
282
282
root := user_model.UserPath(ctxUser.Name)
283
-
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
283
+
if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
284
284
if err != nil {
285
285
if os.IsNotExist(err) {
286
286
return nil
287
287
}
288
288
return err
289
289
}
290
-
if !info.IsDir() || path == root {
290
+
if !d.IsDir() || path == root {
291
291
return nil
292
292
}
293
-
name := info.Name()
293
+
name := d.Name()
294
294
if !strings.HasSuffix(name, ".git") {
295
295
return filepath.SkipDir
296
296
}
···
304
304
count++
305
305
return filepath.SkipDir
306
306
}); err != nil {
307
-
ctx.ServerError("filepath.Walk", err)
307
+
ctx.ServerError("filepath.WalkDir", err)
308
308
return
309
309
}
310
310
+6
-6
services/repository/adopt.go
+6
-6
services/repository/adopt.go
···
303
303
304
304
// We're going to iterate by pagesize.
305
305
root := filepath.Clean(setting.RepoRootPath)
306
-
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
306
+
if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
307
307
if err != nil {
308
308
return err
309
309
}
310
-
if !info.IsDir() || path == root {
310
+
if !d.IsDir() || path == root {
311
311
return nil
312
312
}
313
+
314
+
name := d.Name()
313
315
314
316
if !strings.ContainsRune(path[len(root)+1:], filepath.Separator) {
315
317
// Got a new user
···
318
320
}
319
321
repoNamesToCheck = repoNamesToCheck[:0]
320
322
321
-
if !globUser.Match(info.Name()) {
323
+
if !globUser.Match(name) {
322
324
return filepath.SkipDir
323
325
}
324
326
325
-
userName = info.Name()
327
+
userName = name
326
328
return nil
327
329
}
328
-
329
-
name := info.Name()
330
330
331
331
if !strings.HasSuffix(name, ".git") {
332
332
return filepath.SkipDir