loading up the forgejo repo on tangled to test page performance
Fix bugs in LFS meta garbage collection (#26122)
This PR
- Fix #26093. Replace `time.Time` with `timeutil.TimeStamp`
- Fix #26135. Add missing `xorm:"extends"` to `CountLFSMetaObject` for
LFS meta object query
- Add a unit test for LFS meta object garbage collection
+71
-7
+4
-5
models/git/lfs.go
+4
-5
models/git/lfs.go
···
6
6
import (
7
7
"context"
8
8
"fmt"
9
-
"time"
10
9
11
10
"code.gitea.io/gitea/models/db"
12
11
"code.gitea.io/gitea/models/perm"
···
370
369
371
370
// IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo
372
371
type IterateLFSMetaObjectsForRepoOptions struct {
373
-
OlderThan time.Time
374
-
UpdatedLessRecentlyThan time.Time
372
+
OlderThan timeutil.TimeStamp
373
+
UpdatedLessRecentlyThan timeutil.TimeStamp
375
374
OrderByUpdated bool
376
375
LoopFunctionAlwaysUpdates bool
377
376
}
···
382
381
batchSize := setting.Database.IterateBufferSize
383
382
engine := db.GetEngine(ctx)
384
383
type CountLFSMetaObject struct {
385
-
Count int64
386
-
LFSMetaObject
384
+
Count int64
385
+
LFSMetaObject `xorm:"extends"`
387
386
}
388
387
389
388
id := int64(0)
+3
-2
services/repository/lfs.go
+3
-2
services/repository/lfs.go
···
15
15
"code.gitea.io/gitea/modules/lfs"
16
16
"code.gitea.io/gitea/modules/log"
17
17
"code.gitea.io/gitea/modules/setting"
18
+
"code.gitea.io/gitea/modules/timeutil"
18
19
)
19
20
20
21
// GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function
···
120
121
//
121
122
// It is likely that a week is potentially excessive but it should definitely be enough that any
122
123
// unassociated LFS object is genuinely unassociated.
123
-
OlderThan: opts.OlderThan,
124
-
UpdatedLessRecentlyThan: opts.UpdatedLessRecentlyThan,
124
+
OlderThan: timeutil.TimeStamp(opts.OlderThan.Unix()),
125
+
UpdatedLessRecentlyThan: timeutil.TimeStamp(opts.UpdatedLessRecentlyThan.Unix()),
125
126
OrderByUpdated: true,
126
127
LoopFunctionAlwaysUpdates: true,
127
128
})
+64
services/repository/lfs_test.go
+64
services/repository/lfs_test.go
···
1
+
// Copyright 2023 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package repository
5
+
6
+
import (
7
+
"bytes"
8
+
"context"
9
+
"testing"
10
+
"time"
11
+
12
+
"code.gitea.io/gitea/models/db"
13
+
git_model "code.gitea.io/gitea/models/git"
14
+
repo_model "code.gitea.io/gitea/models/repo"
15
+
"code.gitea.io/gitea/models/unittest"
16
+
"code.gitea.io/gitea/modules/lfs"
17
+
"code.gitea.io/gitea/modules/setting"
18
+
"code.gitea.io/gitea/modules/storage"
19
+
20
+
"github.com/stretchr/testify/assert"
21
+
)
22
+
23
+
func TestGarbageCollectLFSMetaObjects(t *testing.T) {
24
+
assert.NoError(t, unittest.PrepareTestDatabase())
25
+
26
+
setting.LFS.StartServer = true
27
+
err := storage.Init()
28
+
assert.NoError(t, err)
29
+
30
+
repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1")
31
+
assert.NoError(t, err)
32
+
33
+
// add lfs object
34
+
lfsContent := []byte("gitea1")
35
+
lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent)
36
+
37
+
// gc
38
+
err = GarbageCollectLFSMetaObjects(context.Background(), GarbageCollectLFSMetaObjectsOptions{
39
+
AutoFix: true,
40
+
OlderThan: time.Now().Add(7 * 24 * time.Hour).Add(5 * 24 * time.Hour),
41
+
UpdatedLessRecentlyThan: time.Now().Add(7 * 24 * time.Hour).Add(3 * 24 * time.Hour),
42
+
})
43
+
assert.NoError(t, err)
44
+
45
+
// lfs meta has been deleted
46
+
_, err = git_model.GetLFSMetaObjectByOid(db.DefaultContext, repo.ID, lfsOid)
47
+
assert.ErrorIs(t, err, git_model.ErrLFSObjectNotExist)
48
+
}
49
+
50
+
func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string {
51
+
pointer, err := lfs.GeneratePointer(bytes.NewReader(*content))
52
+
assert.NoError(t, err)
53
+
54
+
_, err = git_model.NewLFSMetaObject(db.DefaultContext, &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repositoryID})
55
+
assert.NoError(t, err)
56
+
contentStore := lfs.NewContentStore()
57
+
exist, err := contentStore.Exists(pointer)
58
+
assert.NoError(t, err)
59
+
if !exist {
60
+
err := contentStore.Put(pointer, bytes.NewReader(*content))
61
+
assert.NoError(t, err)
62
+
}
63
+
return pointer.Oid
64
+
}