loading up the forgejo repo on tangled to test page performance
Merge pull request '[ENHANCEMENT] Improve caching of contributor stats' (#4367) from gusted/contributors-stats-cache into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4367
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
+18
-17
+1
release-notes/9.0.0/4367.md
+1
release-notes/9.0.0/4367.md
···
1
+
The caching of contributor stats was improved (the data used by `/<user>/<repo>/activity/recent-commits`) to use the configured cache TTL from the config (`[cache].ITEM_TTL`) instead of a hardcoded TTL of ten minutes. The computation of this operation is computationally heavy and makes a lot of requests to the database and Git on repositories with a lot of commits. It should be cached for longer than what was previously hardcoded, ten minutes.
+7
-13
services/repository/contributors_graph.go
+7
-13
services/repository/contributors_graph.go
···
22
22
"code.gitea.io/gitea/modules/graceful"
23
23
"code.gitea.io/gitea/modules/json"
24
24
"code.gitea.io/gitea/modules/log"
25
+
"code.gitea.io/gitea/modules/setting"
25
26
api "code.gitea.io/gitea/modules/structs"
26
27
27
28
"gitea.com/go-chi/cache"
28
29
)
29
30
30
-
const (
31
-
contributorStatsCacheKey = "GetContributorStats/%s/%s"
32
-
contributorStatsCacheTimeout int64 = 60 * 10
33
-
)
31
+
const contributorStatsCacheKey = "GetContributorStats/%s/%s"
34
32
35
33
var (
36
34
ErrAwaitGeneration = errors.New("generation took longer than ")
···
211
209
212
210
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
213
211
if err != nil {
214
-
err := fmt.Errorf("OpenRepository: %w", err)
215
-
_ = cache.Put(cacheKey, err, contributorStatsCacheTimeout)
212
+
log.Error("OpenRepository[repo=%q]: %v", repo.FullName(), err)
216
213
return
217
214
}
218
215
defer closer.Close()
···
222
219
}
223
220
extendedCommitStats, err := getExtendedCommitStats(gitRepo, revision)
224
221
if err != nil {
225
-
err := fmt.Errorf("ExtendedCommitStats: %w", err)
226
-
_ = cache.Put(cacheKey, err, contributorStatsCacheTimeout)
222
+
log.Error("getExtendedCommitStats[repo=%q revision=%q]: %v", repo.FullName(), revision, err)
227
223
return
228
224
}
229
225
if len(extendedCommitStats) == 0 {
230
-
err := fmt.Errorf("no commit stats returned for revision '%s'", revision)
231
-
_ = cache.Put(cacheKey, err, contributorStatsCacheTimeout)
226
+
log.Error("No commit stats were returned [repo=%q revision=%q]", repo.FullName(), revision)
232
227
return
233
228
}
234
229
···
312
307
313
308
data, err := json.Marshal(contributorsCommitStats)
314
309
if err != nil {
315
-
err := fmt.Errorf("couldn't marshal the data: %w", err)
316
-
_ = cache.Put(cacheKey, err, contributorStatsCacheTimeout)
310
+
log.Error("json.Marshal[repo=%q revision=%q]: %v", repo.FullName(), revision, err)
317
311
return
318
312
}
319
313
320
314
// Store the data as an string, to make it uniform what data type is returned
321
315
// from caches.
322
-
_ = cache.Put(cacheKey, string(data), contributorStatsCacheTimeout)
316
+
_ = cache.Put(cacheKey, string(data), setting.CacheService.TTLSeconds())
323
317
generateLock.Delete(cacheKey)
324
318
if genDone != nil {
325
319
genDone <- struct{}{}
+10
-4
services/repository/contributors_graph_test.go
+10
-4
services/repository/contributors_graph_test.go
···
6
6
import (
7
7
"slices"
8
8
"testing"
9
+
"time"
9
10
10
11
"code.gitea.io/gitea/models/db"
11
12
repo_model "code.gitea.io/gitea/models/repo"
12
13
"code.gitea.io/gitea/models/unittest"
13
-
"code.gitea.io/gitea/modules/git"
14
14
"code.gitea.io/gitea/modules/json"
15
+
"code.gitea.io/gitea/modules/log"
16
+
"code.gitea.io/gitea/modules/test"
15
17
16
18
"gitea.com/go-chi/cache"
17
19
"github.com/stretchr/testify/assert"
···
27
29
})
28
30
assert.NoError(t, err)
29
31
32
+
lc, cleanup := test.NewLogChecker(log.DEFAULT, log.INFO)
33
+
lc.StopMark(`getExtendedCommitStats[repo="user2/repo2" revision="404ref"]: object does not exist [id: 404ref, rel_path: ]`)
34
+
defer cleanup()
35
+
30
36
generateContributorStats(nil, mockCache, "key", repo, "404ref")
31
-
err, isErr := mockCache.Get("key").(error)
32
-
assert.True(t, isErr)
33
-
assert.ErrorAs(t, err, &git.ErrNotExist{})
37
+
assert.False(t, mockCache.IsExist("key"))
38
+
_, stopped := lc.Check(100 * time.Millisecond)
39
+
assert.True(t, stopped)
34
40
35
41
generateContributorStats(nil, mockCache, "key2", repo, "master")
36
42
dataString, isData := mockCache.Get("key2").(string)