loading up the forgejo repo on tangled to test page performance
Merge pull request 'cron task to cleanup dangling container images with version sha256:*' (#4698) from earl-warren/forgejo:wip-container-cleanup into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4698
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
+383
+3
services/packages/container/cleanup.go
+3
services/packages/container/cleanup.go
+142
services/packages/container/cleanup_sha256.go
+142
services/packages/container/cleanup_sha256.go
···
1
+
// Copyright 2024 The Forgejo Authors. All rights reserved.
2
+
// SPDX-License-Identifier: GPL-3.0-or-later
3
+
4
+
package container
5
+
6
+
import (
7
+
"context"
8
+
"strings"
9
+
"time"
10
+
11
+
"code.gitea.io/gitea/models/db"
12
+
"code.gitea.io/gitea/models/packages"
13
+
"code.gitea.io/gitea/modules/json"
14
+
"code.gitea.io/gitea/modules/log"
15
+
container_module "code.gitea.io/gitea/modules/packages/container"
16
+
)
17
+
18
+
var (
19
+
SHA256BatchSize = 500
20
+
SHA256Log = "cleanup dangling images with a sha256:* version"
21
+
SHA256LogStart = "Start to " + SHA256Log
22
+
SHA256LogFinish = "Finished to " + SHA256Log
23
+
)
24
+
25
+
func CleanupSHA256(ctx context.Context, olderThan time.Duration) error {
26
+
log.Info(SHA256LogStart)
27
+
err := cleanupSHA256(ctx, olderThan)
28
+
log.Info(SHA256LogFinish)
29
+
return err
30
+
}
31
+
32
+
func cleanupSHA256(outerCtx context.Context, olderThan time.Duration) error {
33
+
ctx, committer, err := db.TxContext(outerCtx)
34
+
if err != nil {
35
+
return err
36
+
}
37
+
defer committer.Close()
38
+
39
+
foundAtLeastOneSHA256 := false
40
+
shaToVersionID := make(map[string]int64, 100)
41
+
knownSHA := make(map[string]any, 100)
42
+
43
+
log.Debug("Look for all package_version.version that start with sha256:")
44
+
45
+
old := time.Now().Add(-olderThan).Unix()
46
+
47
+
// Iterate over all container versions in ascending order and store
48
+
// in shaToVersionID all versions with a sha256: prefix. If an index
49
+
// manifest is found, the sha256: digest it references are removed
50
+
// from shaToVersionID. If the sha256: digest found in an index
51
+
// manifest is not already in shaToVersionID, it is stored in
52
+
// knownSHA to be dealt with later.
53
+
//
54
+
// Although it is theoretically possible that a sha256: is uploaded
55
+
// after the index manifest that references it, this is not the
56
+
// normal order of operations. First the sha256: version is uploaded
57
+
// and then the index manifest. When the iteration completes,
58
+
// knownSHA will therefore be empty most of the time and
59
+
// shaToVersionID will only contain unreferenced sha256: versions.
60
+
if err := db.GetEngine(ctx).
61
+
Select("`package_version`.`id`, `package_version`.`lower_version`, `package_version`.`metadata_json`").
62
+
Join("INNER", "`package`", "`package`.`id` = `package_version`.`package_id`").
63
+
Where("`package`.`type` = ? AND `package_version`.`created_unix` < ?", packages.TypeContainer, old).
64
+
OrderBy("`package_version`.`id` ASC").
65
+
Iterate(new(packages.PackageVersion), func(_ int, bean any) error {
66
+
v := bean.(*packages.PackageVersion)
67
+
if strings.HasPrefix(v.LowerVersion, "sha256:") {
68
+
shaToVersionID[v.LowerVersion] = v.ID
69
+
foundAtLeastOneSHA256 = true
70
+
} else if strings.Contains(v.MetadataJSON, `"manifests":[{`) {
71
+
var metadata container_module.Metadata
72
+
if err := json.Unmarshal([]byte(v.MetadataJSON), &metadata); err != nil {
73
+
log.Error("package_version.id = %d package_version.metadata_json %s is not a JSON string containing valid metadata. It was ignored but it is an inconsistency in the database that should be looked at. %v", v.ID, v.MetadataJSON, err)
74
+
return nil
75
+
}
76
+
for _, manifest := range metadata.Manifests {
77
+
if _, ok := shaToVersionID[manifest.Digest]; ok {
78
+
delete(shaToVersionID, manifest.Digest)
79
+
} else {
80
+
knownSHA[manifest.Digest] = true
81
+
}
82
+
}
83
+
}
84
+
return nil
85
+
}); err != nil {
86
+
return err
87
+
}
88
+
89
+
for sha := range knownSHA {
90
+
delete(shaToVersionID, sha)
91
+
}
92
+
93
+
if len(shaToVersionID) == 0 {
94
+
if foundAtLeastOneSHA256 {
95
+
log.Debug("All container images with a version matching sha256:* are referenced by an index manifest")
96
+
} else {
97
+
log.Debug("There are no container images with a version matching sha256:*")
98
+
}
99
+
log.Info("Nothing to cleanup")
100
+
return nil
101
+
}
102
+
103
+
found := len(shaToVersionID)
104
+
105
+
log.Warn("%d container image(s) with a version matching sha256:* are not referenced by an index manifest", found)
106
+
107
+
log.Debug("Deleting unreferenced image versions from `package_version`, `package_file` and `package_property` (%d at a time)", SHA256BatchSize)
108
+
109
+
packageVersionIDs := make([]int64, 0, SHA256BatchSize)
110
+
for _, id := range shaToVersionID {
111
+
packageVersionIDs = append(packageVersionIDs, id)
112
+
}
113
+
114
+
for len(packageVersionIDs) > 0 {
115
+
upper := min(len(packageVersionIDs), SHA256BatchSize)
116
+
versionIDs := packageVersionIDs[0:upper]
117
+
118
+
var packageFileIDs []int64
119
+
if err := db.GetEngine(ctx).Select("id").Table("package_file").In("version_id", versionIDs).Find(&packageFileIDs); err != nil {
120
+
return err
121
+
}
122
+
log.Info("Removing %d entries from `package_file` and `package_property`", len(packageFileIDs))
123
+
if _, err := db.GetEngine(ctx).In("id", packageFileIDs).Delete(&packages.PackageFile{}); err != nil {
124
+
return err
125
+
}
126
+
if _, err := db.GetEngine(ctx).In("ref_id", packageFileIDs).And("ref_type = ?", packages.PropertyTypeFile).Delete(&packages.PackageProperty{}); err != nil {
127
+
return err
128
+
}
129
+
130
+
log.Info("Removing %d entries from `package_version` and `package_property`", upper)
131
+
if _, err := db.GetEngine(ctx).In("id", versionIDs).Delete(&packages.PackageVersion{}); err != nil {
132
+
return err
133
+
}
134
+
if _, err := db.GetEngine(ctx).In("ref_id", versionIDs).And("ref_type = ?", packages.PropertyTypeVersion).Delete(&packages.PackageProperty{}); err != nil {
135
+
return err
136
+
}
137
+
138
+
packageVersionIDs = packageVersionIDs[upper:]
139
+
}
140
+
141
+
return committer.Commit()
142
+
}
+238
tests/integration/api_packages_container_cleanup_sha256_test.go
+238
tests/integration/api_packages_container_cleanup_sha256_test.go
···
1
+
// Copyright 2024 The Forgejo Authors. All rights reserved.
2
+
// SPDX-License-Identifier: GPL-3.0-or-later
3
+
4
+
package integration
5
+
6
+
import (
7
+
"bytes"
8
+
"encoding/base64"
9
+
"fmt"
10
+
"net/http"
11
+
"strings"
12
+
"testing"
13
+
"time"
14
+
15
+
"code.gitea.io/gitea/models/db"
16
+
packages_model "code.gitea.io/gitea/models/packages"
17
+
"code.gitea.io/gitea/models/unittest"
18
+
user_model "code.gitea.io/gitea/models/user"
19
+
"code.gitea.io/gitea/modules/log"
20
+
packages_module "code.gitea.io/gitea/modules/packages"
21
+
"code.gitea.io/gitea/modules/setting"
22
+
"code.gitea.io/gitea/modules/test"
23
+
packages_cleanup "code.gitea.io/gitea/services/packages/cleanup"
24
+
packages_container "code.gitea.io/gitea/services/packages/container"
25
+
"code.gitea.io/gitea/tests"
26
+
27
+
oci "github.com/opencontainers/image-spec/specs-go/v1"
28
+
"github.com/stretchr/testify/assert"
29
+
"github.com/stretchr/testify/require"
30
+
)
31
+
32
+
func TestPackagesContainerCleanupSHA256(t *testing.T) {
33
+
defer tests.PrepareTestEnv(t, 1)()
34
+
defer test.MockVariableValue(&setting.Packages.Storage.Type, setting.LocalStorageType)()
35
+
defer test.MockVariableValue(&packages_container.SHA256BatchSize, 1)()
36
+
37
+
ctx := db.DefaultContext
38
+
39
+
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
40
+
41
+
cleanupAndCheckLogs := func(t *testing.T, expected ...string) {
42
+
t.Helper()
43
+
logChecker, cleanup := test.NewLogChecker(log.DEFAULT, log.TRACE)
44
+
logChecker.Filter(expected...)
45
+
logChecker.StopMark(packages_container.SHA256LogFinish)
46
+
defer cleanup()
47
+
48
+
require.NoError(t, packages_cleanup.CleanupExpiredData(ctx, -1*time.Hour))
49
+
50
+
logFiltered, logStopped := logChecker.Check(5 * time.Second)
51
+
assert.True(t, logStopped)
52
+
filtered := make([]bool, 0, len(expected))
53
+
for range expected {
54
+
filtered = append(filtered, true)
55
+
}
56
+
assert.EqualValues(t, filtered, logFiltered, expected)
57
+
}
58
+
59
+
userToken := ""
60
+
61
+
t.Run("Authenticate", func(t *testing.T) {
62
+
type TokenResponse struct {
63
+
Token string `json:"token"`
64
+
}
65
+
66
+
authenticate := []string{`Bearer realm="` + setting.AppURL + `v2/token",service="container_registry",scope="*"`}
67
+
68
+
t.Run("User", func(t *testing.T) {
69
+
req := NewRequest(t, "GET", fmt.Sprintf("%sv2", setting.AppURL))
70
+
resp := MakeRequest(t, req, http.StatusUnauthorized)
71
+
72
+
assert.ElementsMatch(t, authenticate, resp.Header().Values("WWW-Authenticate"))
73
+
74
+
req = NewRequest(t, "GET", fmt.Sprintf("%sv2/token", setting.AppURL)).
75
+
AddBasicAuth(user.Name)
76
+
resp = MakeRequest(t, req, http.StatusOK)
77
+
78
+
tokenResponse := &TokenResponse{}
79
+
DecodeJSON(t, resp, &tokenResponse)
80
+
81
+
assert.NotEmpty(t, tokenResponse.Token)
82
+
83
+
userToken = fmt.Sprintf("Bearer %s", tokenResponse.Token)
84
+
85
+
req = NewRequest(t, "GET", fmt.Sprintf("%sv2", setting.AppURL)).
86
+
AddTokenAuth(userToken)
87
+
MakeRequest(t, req, http.StatusOK)
88
+
})
89
+
})
90
+
91
+
image := "test"
92
+
multiTag := "multi"
93
+
94
+
url := fmt.Sprintf("%sv2/%s/%s", setting.AppURL, user.Name, image)
95
+
96
+
blobDigest := "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
97
+
sha256ManifestDigest := "sha256:4305f5f5572b9a426b88909b036e52ee3cf3d7b9c1b01fac840e90747f56623d"
98
+
indexManifestDigest := "sha256:b992f98104ab25f60d78368a674ce6f6a49741f4e32729e8496067ed06174e9b"
99
+
100
+
uploadSHA256Version := func(t *testing.T) {
101
+
t.Helper()
102
+
103
+
blobContent, _ := base64.StdEncoding.DecodeString(`H4sIAAAJbogA/2IYBaNgFIxYAAgAAP//Lq+17wAEAAA=`)
104
+
105
+
req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, blobDigest), bytes.NewReader(blobContent)).
106
+
AddTokenAuth(userToken)
107
+
resp := MakeRequest(t, req, http.StatusCreated)
108
+
109
+
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
110
+
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
111
+
112
+
configDigest := "sha256:4607e093bec406eaadb6f3a340f63400c9d3a7038680744c406903766b938f0d"
113
+
configContent := `{"architecture":"amd64","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/true"],"ArgsEscaped":true,"Image":"sha256:9bd8b88dc68b80cffe126cc820e4b52c6e558eb3b37680bfee8e5f3ed7b8c257"},"container":"b89fe92a887d55c0961f02bdfbfd8ac3ddf66167db374770d2d9e9fab3311510","container_config":{"Hostname":"b89fe92a887d","Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/true\"]"],"ArgsEscaped":true,"Image":"sha256:9bd8b88dc68b80cffe126cc820e4b52c6e558eb3b37680bfee8e5f3ed7b8c257"},"created":"2022-01-01T00:00:00.000000000Z","docker_version":"20.10.12","history":[{"created":"2022-01-01T00:00:00.000000000Z","created_by":"/bin/sh -c #(nop) COPY file:0e7589b0c800daaf6fa460d2677101e4676dd9491980210cb345480e513f3602 in /true "},{"created":"2022-01-01T00:00:00.000000001Z","created_by":"/bin/sh -c #(nop) CMD [\"/true\"]","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:0ff3b91bdf21ecdf2f2f3d4372c2098a14dbe06cd678e8f0a85fd4902d00e2e2"]}}`
114
+
115
+
req = NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, configDigest), strings.NewReader(configContent)).
116
+
AddTokenAuth(userToken)
117
+
MakeRequest(t, req, http.StatusCreated)
118
+
119
+
sha256ManifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeImageManifest + `","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:4607e093bec406eaadb6f3a340f63400c9d3a7038680744c406903766b938f0d","size":1069},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4","size":32}]}`
120
+
req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, sha256ManifestDigest), strings.NewReader(sha256ManifestContent)).
121
+
AddTokenAuth(userToken).
122
+
SetHeader("Content-Type", oci.MediaTypeImageManifest)
123
+
resp = MakeRequest(t, req, http.StatusCreated)
124
+
125
+
assert.Equal(t, sha256ManifestDigest, resp.Header().Get("Docker-Content-Digest"))
126
+
127
+
req = NewRequest(t, "HEAD", fmt.Sprintf("%s/manifests/%s", url, sha256ManifestDigest)).
128
+
AddTokenAuth(userToken)
129
+
resp = MakeRequest(t, req, http.StatusOK)
130
+
131
+
assert.Equal(t, fmt.Sprintf("%d", len(sha256ManifestContent)), resp.Header().Get("Content-Length"))
132
+
assert.Equal(t, sha256ManifestDigest, resp.Header().Get("Docker-Content-Digest"))
133
+
}
134
+
135
+
uploadIndexManifest := func(t *testing.T) {
136
+
indexManifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeImageIndex + `","manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"` + sha256ManifestDigest + `","platform":{"os":"linux","architecture":"arm","variant":"v7"}}]}`
137
+
req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, multiTag), strings.NewReader(indexManifestContent)).
138
+
AddTokenAuth(userToken).
139
+
SetHeader("Content-Type", oci.MediaTypeImageIndex)
140
+
resp := MakeRequest(t, req, http.StatusCreated)
141
+
142
+
assert.Equal(t, indexManifestDigest, resp.Header().Get("Docker-Content-Digest"))
143
+
}
144
+
145
+
assertImageExists := func(t *testing.T, manifestDigest, blobDigest string) {
146
+
req := NewRequest(t, "HEAD", fmt.Sprintf("%s/manifests/%s", url, manifestDigest)).
147
+
AddTokenAuth(userToken)
148
+
MakeRequest(t, req, http.StatusOK)
149
+
150
+
req = NewRequest(t, "HEAD", fmt.Sprintf("%s/blobs/%s", url, blobDigest)).
151
+
AddTokenAuth(userToken)
152
+
MakeRequest(t, req, http.StatusOK)
153
+
}
154
+
155
+
assertImageNotExists := func(t *testing.T, manifestDigest, blobDigest string) {
156
+
req := NewRequest(t, "HEAD", fmt.Sprintf("%s/manifests/%s", url, manifestDigest)).
157
+
AddTokenAuth(userToken)
158
+
MakeRequest(t, req, http.StatusNotFound)
159
+
160
+
req = NewRequest(t, "HEAD", fmt.Sprintf("%s/blobs/%s", url, blobDigest)).
161
+
AddTokenAuth(userToken)
162
+
MakeRequest(t, req, http.StatusNotFound)
163
+
}
164
+
165
+
assertImageDeleted := func(t *testing.T, image, manifestDigest, blobDigest string, cleanup func()) {
166
+
t.Helper()
167
+
packageVersion := unittest.AssertExistsAndLoadBean(t, &packages_model.PackageVersion{Version: manifestDigest})
168
+
packageFile := unittest.AssertExistsAndLoadBean(t, &packages_model.PackageFile{VersionID: packageVersion.ID})
169
+
unittest.AssertExistsAndLoadBean(t, &packages_model.PackageProperty{RefID: packageFile.ID, RefType: packages_model.PropertyTypeVersion})
170
+
packageBlob := unittest.AssertExistsAndLoadBean(t, &packages_model.PackageBlob{ID: packageFile.BlobID})
171
+
contentStore := packages_module.NewContentStore()
172
+
require.NoError(t, contentStore.Has(packages_module.BlobHash256Key(packageBlob.HashSHA256)))
173
+
174
+
assertImageExists(t, manifestDigest, blobDigest)
175
+
176
+
cleanup()
177
+
178
+
assertImageNotExists(t, manifestDigest, blobDigest)
179
+
180
+
unittest.AssertNotExistsBean(t, &packages_model.PackageVersion{Version: manifestDigest})
181
+
unittest.AssertNotExistsBean(t, &packages_model.PackageFile{VersionID: packageVersion.ID})
182
+
unittest.AssertNotExistsBean(t, &packages_model.PackageProperty{RefID: packageFile.ID, RefType: packages_model.PropertyTypeVersion})
183
+
unittest.AssertNotExistsBean(t, &packages_model.PackageBlob{ID: packageFile.BlobID})
184
+
assert.Error(t, contentStore.Has(packages_module.BlobHash256Key(packageBlob.HashSHA256)))
185
+
}
186
+
187
+
assertImageAndPackageDeleted := func(t *testing.T, image, manifestDigest, blobDigest string, cleanup func()) {
188
+
t.Helper()
189
+
unittest.AssertExistsAndLoadBean(t, &packages_model.Package{Name: image})
190
+
assertImageDeleted(t, image, manifestDigest, blobDigest, cleanup)
191
+
unittest.AssertNotExistsBean(t, &packages_model.Package{Name: image})
192
+
}
193
+
194
+
t.Run("Nothing to look at", func(t *testing.T) {
195
+
cleanupAndCheckLogs(t, "There are no container images with a version matching sha256:*")
196
+
})
197
+
198
+
uploadSHA256Version(t)
199
+
200
+
t.Run("Dangling image found", func(t *testing.T) {
201
+
assertImageAndPackageDeleted(t, image, sha256ManifestDigest, blobDigest, func() {
202
+
cleanupAndCheckLogs(t,
203
+
"Removing 3 entries from `package_file` and `package_property`",
204
+
"Removing 1 entries from `package_version` and `package_property`",
205
+
)
206
+
})
207
+
})
208
+
209
+
uploadSHA256Version(t)
210
+
uploadIndexManifest(t)
211
+
212
+
t.Run("Corrupted index manifest metadata is ignored", func(t *testing.T) {
213
+
assertImageExists(t, sha256ManifestDigest, blobDigest)
214
+
_, err := db.GetEngine(ctx).Table("package_version").Where("version = ?", multiTag).Update(&packages_model.PackageVersion{MetadataJSON: `corrupted "manifests":[{ bad`})
215
+
require.NoError(t, err)
216
+
217
+
// do not expect the package to be deleted because it contains
218
+
// corrupted metadata that prevents that from happening
219
+
assertImageDeleted(t, image, sha256ManifestDigest, blobDigest, func() {
220
+
cleanupAndCheckLogs(t,
221
+
"Removing 3 entries from `package_file` and `package_property`",
222
+
"Removing 1 entries from `package_version` and `package_property`",
223
+
"is not a JSON string containing valid metadata",
224
+
)
225
+
})
226
+
})
227
+
228
+
uploadSHA256Version(t)
229
+
uploadIndexManifest(t)
230
+
231
+
t.Run("Image found but referenced", func(t *testing.T) {
232
+
assertImageExists(t, sha256ManifestDigest, blobDigest)
233
+
cleanupAndCheckLogs(t,
234
+
"All container images with a version matching sha256:* are referenced by an index manifest",
235
+
)
236
+
assertImageExists(t, sha256ManifestDigest, blobDigest)
237
+
})
238
+
}