loading up the forgejo repo on tangled to test page performance
Implement actions artifacts (#22738)
Implement action artifacts server api.
This change is used for supporting
https://github.com/actions/upload-artifact and
https://github.com/actions/download-artifact in gitea actions. It can
run sample workflow from doc
https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts.
The api design is inspired by
https://github.com/nektos/act/blob/master/pkg/artifacts/server.go and
includes some changes from gitea internal structs and methods.
Actions artifacts contains two parts:
- Gitea server api and storage (this pr implement basic design without
some complex cases supports)
- Runner communicate with gitea server api (in comming)
Old pr https://github.com/go-gitea/gitea/pull/22345 is outdated after
actions merged. I create new pr from main branch.
Add artifacts list in actions workflow page.
+1127
-6
+122
models/actions/artifact.go
+122
models/actions/artifact.go
···
1
+
// Copyright 2023 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
// This artifact server is inspired by https://github.com/nektos/act/blob/master/pkg/artifacts/server.go.
5
+
// It updates url setting and uses ObjectStore to handle artifacts persistence.
6
+
7
+
package actions
8
+
9
+
import (
10
+
"context"
11
+
"errors"
12
+
13
+
"code.gitea.io/gitea/models/db"
14
+
"code.gitea.io/gitea/modules/timeutil"
15
+
"code.gitea.io/gitea/modules/util"
16
+
)
17
+
18
+
const (
19
+
// ArtifactStatusUploadPending is the status of an artifact upload that is pending
20
+
ArtifactStatusUploadPending = 1
21
+
// ArtifactStatusUploadConfirmed is the status of an artifact upload that is confirmed
22
+
ArtifactStatusUploadConfirmed = 2
23
+
// ArtifactStatusUploadError is the status of an artifact upload that is errored
24
+
ArtifactStatusUploadError = 3
25
+
)
26
+
27
+
func init() {
28
+
db.RegisterModel(new(ActionArtifact))
29
+
}
30
+
31
+
// ActionArtifact is a file that is stored in the artifact storage.
32
+
type ActionArtifact struct {
33
+
ID int64 `xorm:"pk autoincr"`
34
+
RunID int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact
35
+
RunnerID int64
36
+
RepoID int64 `xorm:"index"`
37
+
OwnerID int64
38
+
CommitSHA string
39
+
StoragePath string // The path to the artifact in the storage
40
+
FileSize int64 // The size of the artifact in bytes
41
+
FileCompressedSize int64 // The size of the artifact in bytes after gzip compression
42
+
ContentEncoding string // The content encoding of the artifact
43
+
ArtifactPath string // The path to the artifact when runner uploads it
44
+
ArtifactName string `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it
45
+
Status int64 `xorm:"index"` // The status of the artifact, uploading, expired or need-delete
46
+
CreatedUnix timeutil.TimeStamp `xorm:"created"`
47
+
UpdatedUnix timeutil.TimeStamp `xorm:"updated index"`
48
+
}
49
+
50
+
// CreateArtifact create a new artifact with task info or get same named artifact in the same run
51
+
func CreateArtifact(ctx context.Context, t *ActionTask, artifactName string) (*ActionArtifact, error) {
52
+
if err := t.LoadJob(ctx); err != nil {
53
+
return nil, err
54
+
}
55
+
artifact, err := getArtifactByArtifactName(ctx, t.Job.RunID, artifactName)
56
+
if errors.Is(err, util.ErrNotExist) {
57
+
artifact := &ActionArtifact{
58
+
RunID: t.Job.RunID,
59
+
RunnerID: t.RunnerID,
60
+
RepoID: t.RepoID,
61
+
OwnerID: t.OwnerID,
62
+
CommitSHA: t.CommitSHA,
63
+
Status: ArtifactStatusUploadPending,
64
+
}
65
+
if _, err := db.GetEngine(ctx).Insert(artifact); err != nil {
66
+
return nil, err
67
+
}
68
+
return artifact, nil
69
+
} else if err != nil {
70
+
return nil, err
71
+
}
72
+
return artifact, nil
73
+
}
74
+
75
+
func getArtifactByArtifactName(ctx context.Context, runID int64, name string) (*ActionArtifact, error) {
76
+
var art ActionArtifact
77
+
has, err := db.GetEngine(ctx).Where("run_id = ? AND artifact_name = ?", runID, name).Get(&art)
78
+
if err != nil {
79
+
return nil, err
80
+
} else if !has {
81
+
return nil, util.ErrNotExist
82
+
}
83
+
return &art, nil
84
+
}
85
+
86
+
// GetArtifactByID returns an artifact by id
87
+
func GetArtifactByID(ctx context.Context, id int64) (*ActionArtifact, error) {
88
+
var art ActionArtifact
89
+
has, err := db.GetEngine(ctx).ID(id).Get(&art)
90
+
if err != nil {
91
+
return nil, err
92
+
} else if !has {
93
+
return nil, util.ErrNotExist
94
+
}
95
+
96
+
return &art, nil
97
+
}
98
+
99
+
// UpdateArtifactByID updates an artifact by id
100
+
func UpdateArtifactByID(ctx context.Context, id int64, art *ActionArtifact) error {
101
+
art.ID = id
102
+
_, err := db.GetEngine(ctx).ID(id).AllCols().Update(art)
103
+
return err
104
+
}
105
+
106
+
// ListArtifactsByRunID returns all artifacts of a run
107
+
func ListArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) {
108
+
arts := make([]*ActionArtifact, 0, 10)
109
+
return arts, db.GetEngine(ctx).Where("run_id=?", runID).Find(&arts)
110
+
}
111
+
112
+
// ListUploadedArtifactsByRunID returns all uploaded artifacts of a run
113
+
func ListUploadedArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) {
114
+
arts := make([]*ActionArtifact, 0, 10)
115
+
return arts, db.GetEngine(ctx).Where("run_id=? AND status=?", runID, ArtifactStatusUploadConfirmed).Find(&arts)
116
+
}
117
+
118
+
// ListArtifactsByRepoID returns all artifacts of a repo
119
+
func ListArtifactsByRepoID(ctx context.Context, repoID int64) ([]*ActionArtifact, error) {
120
+
arts := make([]*ActionArtifact, 0, 10)
121
+
return arts, db.GetEngine(ctx).Where("repo_id=?", repoID).Find(&arts)
122
+
}
+1
-1
models/fixtures/access_token.yml
+1
-1
models/fixtures/access_token.yml
+19
models/fixtures/action_run.yml
+19
models/fixtures/action_run.yml
···
1
+
-
2
+
id: 791
3
+
title: "update actions"
4
+
repo_id: 4
5
+
owner_id: 1
6
+
workflow_id: "artifact.yaml"
7
+
index: 187
8
+
trigger_user_id: 1
9
+
ref: "refs/heads/master"
10
+
commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0"
11
+
event: "push"
12
+
is_fork_pull_request: 0
13
+
status: 1
14
+
started: 1683636528
15
+
stopped: 1683636626
16
+
created: 1683636108
17
+
updated: 1683636626
18
+
need_approval: 0
19
+
approved_by: 0
+14
models/fixtures/action_run_job.yml
+14
models/fixtures/action_run_job.yml
+20
models/fixtures/action_task.yml
+20
models/fixtures/action_task.yml
···
1
+
-
2
+
id: 47
3
+
job_id: 192
4
+
attempt: 3
5
+
runner_id: 1
6
+
status: 6 # 6 is the status code for "running", running task can upload artifacts
7
+
started: 1683636528
8
+
stopped: 1683636626
9
+
repo_id: 4
10
+
owner_id: 1
11
+
commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
12
+
is_fork_pull_request: 0
13
+
token_hash: 6d8ef48297195edcc8e22c70b3020eaa06c52976db67d39b4260c64a69a2cc1508825121b7b8394e48e00b1bf8718b2a867e
14
+
token_salt: jVuKnSPGgy
15
+
token_last_eight: eeb1a71a
16
+
log_filename: artifact-test2/2f/47.log
17
+
log_in_storage: 1
18
+
log_length: 707
19
+
log_size: 90179
20
+
log_expired: 0
+2
models/migrations/migrations.go
+2
models/migrations/migrations.go
···
491
491
NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository),
492
492
// v256 -> v257
493
493
NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage),
494
+
// v257 -> v258
495
+
NewMigration("Add Actions Artifact table", v1_20.CreateActionArtifactTable),
494
496
}
495
497
496
498
// GetCurrentDBVersion returns the current db version
+33
models/migrations/v1_20/v257.go
+33
models/migrations/v1_20/v257.go
···
1
+
// Copyright 2023 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package v1_20 //nolint
5
+
6
+
import (
7
+
"code.gitea.io/gitea/modules/timeutil"
8
+
9
+
"xorm.io/xorm"
10
+
)
11
+
12
+
func CreateActionArtifactTable(x *xorm.Engine) error {
13
+
// ActionArtifact is a file that is stored in the artifact storage.
14
+
type ActionArtifact struct {
15
+
ID int64 `xorm:"pk autoincr"`
16
+
RunID int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact
17
+
RunnerID int64
18
+
RepoID int64 `xorm:"index"`
19
+
OwnerID int64
20
+
CommitSHA string
21
+
StoragePath string // The path to the artifact in the storage
22
+
FileSize int64 // The size of the artifact in bytes
23
+
FileCompressedSize int64 // The size of the artifact in bytes after gzip compression
24
+
ContentEncoding string // The content encoding of the artifact
25
+
ArtifactPath string // The path to the artifact when runner uploads it
26
+
ArtifactName string `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it
27
+
Status int64 `xorm:"index"` // The status of the artifact
28
+
CreatedUnix timeutil.TimeStamp `xorm:"created"`
29
+
UpdatedUnix timeutil.TimeStamp `xorm:"updated index"`
30
+
}
31
+
32
+
return x.Sync(new(ActionArtifact))
33
+
}
+15
models/repo.go
+15
models/repo.go
···
59
59
return fmt.Errorf("find actions tasks of repo %v: %w", repoID, err)
60
60
}
61
61
62
+
// Query the artifacts of this repo, they will be needed after they have been deleted to remove artifacts files in ObjectStorage
63
+
artifacts, err := actions_model.ListArtifactsByRepoID(ctx, repoID)
64
+
if err != nil {
65
+
return fmt.Errorf("list actions artifacts of repo %v: %w", repoID, err)
66
+
}
67
+
62
68
// In case is a organization.
63
69
org, err := user_model.GetUserByID(ctx, uid)
64
70
if err != nil {
···
164
170
&actions_model.ActionRunJob{RepoID: repoID},
165
171
&actions_model.ActionRun{RepoID: repoID},
166
172
&actions_model.ActionRunner{RepoID: repoID},
173
+
&actions_model.ActionArtifact{RepoID: repoID},
167
174
); err != nil {
168
175
return fmt.Errorf("deleteBeans: %w", err)
169
176
}
···
332
339
err := actions_module.RemoveLogs(ctx, task.LogInStorage, task.LogFilename)
333
340
if err != nil {
334
341
log.Error("remove log file %q: %v", task.LogFilename, err)
342
+
// go on
343
+
}
344
+
}
345
+
346
+
// delete actions artifacts in ObjectStorage after the repo have already been deleted
347
+
for _, art := range artifacts {
348
+
if err := storage.ActionsArtifacts.Delete(art.StoragePath); err != nil {
349
+
log.Error("remove artifact file %q: %v", art.StoragePath, err)
335
350
// go on
336
351
}
337
352
}
+1
-1
models/unittest/testdb.go
+1
-1
models/unittest/testdb.go
···
126
126
127
127
setting.Packages.Storage.Path = filepath.Join(setting.AppDataPath, "packages")
128
128
129
-
setting.Actions.Storage.Path = filepath.Join(setting.AppDataPath, "actions_log")
129
+
setting.Actions.LogStorage.Path = filepath.Join(setting.AppDataPath, "actions_log")
130
130
131
131
setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home")
132
132
+7
-2
modules/setting/actions.go
+7
-2
modules/setting/actions.go
···
10
10
// Actions settings
11
11
var (
12
12
Actions = struct {
13
-
Storage // how the created logs should be stored
13
+
LogStorage Storage // how the created logs should be stored
14
+
ArtifactStorage Storage // how the created artifacts should be stored
14
15
Enabled bool
15
16
DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"`
16
17
}{
···
25
26
log.Fatal("Failed to map Actions settings: %v", err)
26
27
}
27
28
28
-
Actions.Storage = getStorage(rootCfg, "actions_log", "", nil)
29
+
actionsSec := rootCfg.Section("actions.artifacts")
30
+
storageType := actionsSec.Key("STORAGE_TYPE").MustString("")
31
+
32
+
Actions.LogStorage = getStorage(rootCfg, "actions_log", "", nil)
33
+
Actions.ArtifactStorage = getStorage(rootCfg, "actions_artifacts", storageType, actionsSec)
29
34
}
+9
-2
modules/storage/storage.go
+9
-2
modules/storage/storage.go
···
128
128
129
129
// Actions represents actions storage
130
130
Actions ObjectStorage = uninitializedStorage
131
+
// Actions Artifacts represents actions artifacts storage
132
+
ActionsArtifacts ObjectStorage = uninitializedStorage
131
133
)
132
134
133
135
// Init init the stoarge
···
212
214
func initActions() (err error) {
213
215
if !setting.Actions.Enabled {
214
216
Actions = discardStorage("Actions isn't enabled")
217
+
ActionsArtifacts = discardStorage("ActionsArtifacts isn't enabled")
215
218
return nil
216
219
}
217
-
log.Info("Initialising Actions storage with type: %s", setting.Actions.Storage.Type)
218
-
Actions, err = NewStorage(setting.Actions.Storage.Type, &setting.Actions.Storage)
220
+
log.Info("Initialising Actions storage with type: %s", setting.Actions.LogStorage.Type)
221
+
if Actions, err = NewStorage(setting.Actions.LogStorage.Type, &setting.Actions.LogStorage); err != nil {
222
+
return err
223
+
}
224
+
log.Info("Initialising ActionsArtifacts storage with type: %s", setting.Actions.ArtifactStorage.Type)
225
+
ActionsArtifacts, err = NewStorage(setting.Actions.ArtifactStorage.Type, &setting.Actions.ArtifactStorage)
219
226
return err
220
227
}
+2
options/locale/locale_en-US.ini
+2
options/locale/locale_en-US.ini
+587
routers/api/actions/artifacts.go
+587
routers/api/actions/artifacts.go
···
1
+
// Copyright 2023 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package actions
5
+
6
+
// Github Actions Artifacts API Simple Description
7
+
//
8
+
// 1. Upload artifact
9
+
// 1.1. Post upload url
10
+
// Post: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts?api-version=6.0-preview
11
+
// Request:
12
+
// {
13
+
// "Type": "actions_storage",
14
+
// "Name": "artifact"
15
+
// }
16
+
// Response:
17
+
// {
18
+
// "fileContainerResourceUrl":"/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload"
19
+
// }
20
+
// it acquires an upload url for artifact upload
21
+
// 1.2. Upload artifact
22
+
// PUT: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload?itemPath=artifact%2Ffilename
23
+
// it upload chunk with headers:
24
+
// x-tfs-filelength: 1024 // total file length
25
+
// content-length: 1024 // chunk length
26
+
// x-actions-results-md5: md5sum // md5sum of chunk
27
+
// content-range: bytes 0-1023/1024 // chunk range
28
+
// we save all chunks to one storage directory after md5sum check
29
+
// 1.3. Confirm upload
30
+
// PATCH: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload?itemPath=artifact%2Ffilename
31
+
// it confirm upload and merge all chunks to one file, save this file to storage
32
+
//
33
+
// 2. Download artifact
34
+
// 2.1 list artifacts
35
+
// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts?api-version=6.0-preview
36
+
// Response:
37
+
// {
38
+
// "count": 1,
39
+
// "value": [
40
+
// {
41
+
// "name": "artifact",
42
+
// "fileContainerResourceUrl": "/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/path"
43
+
// }
44
+
// ]
45
+
// }
46
+
// 2.2 download artifact
47
+
// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/path?api-version=6.0-preview
48
+
// Response:
49
+
// {
50
+
// "value": [
51
+
// {
52
+
// "contentLocation": "/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/download",
53
+
// "path": "artifact/filename",
54
+
// "itemType": "file"
55
+
// }
56
+
// ]
57
+
// }
58
+
// 2.3 download artifact file
59
+
// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/download?itemPath=artifact%2Ffilename
60
+
// Response:
61
+
// download file
62
+
//
63
+
64
+
import (
65
+
"compress/gzip"
66
+
gocontext "context"
67
+
"crypto/md5"
68
+
"encoding/base64"
69
+
"errors"
70
+
"fmt"
71
+
"io"
72
+
"net/http"
73
+
"sort"
74
+
"strconv"
75
+
"strings"
76
+
"time"
77
+
78
+
"code.gitea.io/gitea/models/actions"
79
+
"code.gitea.io/gitea/modules/context"
80
+
"code.gitea.io/gitea/modules/json"
81
+
"code.gitea.io/gitea/modules/log"
82
+
"code.gitea.io/gitea/modules/setting"
83
+
"code.gitea.io/gitea/modules/storage"
84
+
"code.gitea.io/gitea/modules/util"
85
+
"code.gitea.io/gitea/modules/web"
86
+
)
87
+
88
+
const (
89
+
artifactXTfsFileLengthHeader = "x-tfs-filelength"
90
+
artifactXActionsResultsMD5Header = "x-actions-results-md5"
91
+
)
92
+
93
+
const artifactRouteBase = "/_apis/pipelines/workflows/{run_id}/artifacts"
94
+
95
+
func ArtifactsRoutes(goctx gocontext.Context, prefix string) *web.Route {
96
+
m := web.NewRoute()
97
+
m.Use(withContexter(goctx))
98
+
99
+
r := artifactRoutes{
100
+
prefix: prefix,
101
+
fs: storage.ActionsArtifacts,
102
+
}
103
+
104
+
m.Group(artifactRouteBase, func() {
105
+
// retrieve, list and confirm artifacts
106
+
m.Combo("").Get(r.listArtifacts).Post(r.getUploadArtifactURL).Patch(r.comfirmUploadArtifact)
107
+
// handle container artifacts list and download
108
+
m.Group("/{artifact_id}", func() {
109
+
m.Put("/upload", r.uploadArtifact)
110
+
m.Get("/path", r.getDownloadArtifactURL)
111
+
m.Get("/download", r.downloadArtifact)
112
+
})
113
+
})
114
+
115
+
return m
116
+
}
117
+
118
+
// withContexter initializes a package context for a request.
119
+
func withContexter(goctx gocontext.Context) func(next http.Handler) http.Handler {
120
+
return func(next http.Handler) http.Handler {
121
+
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
122
+
ctx := context.Context{
123
+
Resp: context.NewResponse(resp),
124
+
Data: map[string]interface{}{},
125
+
}
126
+
defer ctx.Close()
127
+
128
+
// action task call server api with Bearer ACTIONS_RUNTIME_TOKEN
129
+
// we should verify the ACTIONS_RUNTIME_TOKEN
130
+
authHeader := req.Header.Get("Authorization")
131
+
if len(authHeader) == 0 || !strings.HasPrefix(authHeader, "Bearer ") {
132
+
ctx.Error(http.StatusUnauthorized, "Bad authorization header")
133
+
return
134
+
}
135
+
authToken := strings.TrimPrefix(authHeader, "Bearer ")
136
+
task, err := actions.GetRunningTaskByToken(req.Context(), authToken)
137
+
if err != nil {
138
+
log.Error("Error runner api getting task: %v", err)
139
+
ctx.Error(http.StatusInternalServerError, "Error runner api getting task")
140
+
return
141
+
}
142
+
ctx.Data["task"] = task
143
+
144
+
if err := task.LoadJob(goctx); err != nil {
145
+
log.Error("Error runner api getting job: %v", err)
146
+
ctx.Error(http.StatusInternalServerError, "Error runner api getting job")
147
+
return
148
+
}
149
+
150
+
ctx.Req = context.WithContext(req, &ctx)
151
+
152
+
next.ServeHTTP(ctx.Resp, ctx.Req)
153
+
})
154
+
}
155
+
}
156
+
157
+
type artifactRoutes struct {
158
+
prefix string
159
+
fs storage.ObjectStorage
160
+
}
161
+
162
+
func (ar artifactRoutes) buildArtifactURL(runID, artifactID int64, suffix string) string {
163
+
uploadURL := strings.TrimSuffix(setting.AppURL, "/") + strings.TrimSuffix(ar.prefix, "/") +
164
+
strings.ReplaceAll(artifactRouteBase, "{run_id}", strconv.FormatInt(runID, 10)) +
165
+
"/" + strconv.FormatInt(artifactID, 10) + "/" + suffix
166
+
return uploadURL
167
+
}
168
+
169
+
type getUploadArtifactRequest struct {
170
+
Type string
171
+
Name string
172
+
}
173
+
174
+
type getUploadArtifactResponse struct {
175
+
FileContainerResourceURL string `json:"fileContainerResourceUrl"`
176
+
}
177
+
178
+
func (ar artifactRoutes) validateRunID(ctx *context.Context) (*actions.ActionTask, int64, bool) {
179
+
task, ok := ctx.Data["task"].(*actions.ActionTask)
180
+
if !ok {
181
+
log.Error("Error getting task in context")
182
+
ctx.Error(http.StatusInternalServerError, "Error getting task in context")
183
+
return nil, 0, false
184
+
}
185
+
runID := ctx.ParamsInt64("run_id")
186
+
if task.Job.RunID != runID {
187
+
log.Error("Error runID not match")
188
+
ctx.Error(http.StatusBadRequest, "run-id does not match")
189
+
return nil, 0, false
190
+
}
191
+
return task, runID, true
192
+
}
193
+
194
+
// getUploadArtifactURL generates a URL for uploading an artifact
195
+
func (ar artifactRoutes) getUploadArtifactURL(ctx *context.Context) {
196
+
task, runID, ok := ar.validateRunID(ctx)
197
+
if !ok {
198
+
return
199
+
}
200
+
201
+
var req getUploadArtifactRequest
202
+
if err := json.NewDecoder(ctx.Req.Body).Decode(&req); err != nil {
203
+
log.Error("Error decode request body: %v", err)
204
+
ctx.Error(http.StatusInternalServerError, "Error decode request body")
205
+
return
206
+
}
207
+
208
+
artifact, err := actions.CreateArtifact(ctx, task, req.Name)
209
+
if err != nil {
210
+
log.Error("Error creating artifact: %v", err)
211
+
ctx.Error(http.StatusInternalServerError, err.Error())
212
+
return
213
+
}
214
+
resp := getUploadArtifactResponse{
215
+
FileContainerResourceURL: ar.buildArtifactURL(runID, artifact.ID, "upload"),
216
+
}
217
+
log.Debug("[artifact] get upload url: %s, artifact id: %d", resp.FileContainerResourceURL, artifact.ID)
218
+
ctx.JSON(http.StatusOK, resp)
219
+
}
220
+
221
+
// getUploadFileSize returns the size of the file to be uploaded.
222
+
// The raw size is the size of the file as reported by the header X-TFS-FileLength.
223
+
func (ar artifactRoutes) getUploadFileSize(ctx *context.Context) (int64, int64, error) {
224
+
contentLength := ctx.Req.ContentLength
225
+
xTfsLength, _ := strconv.ParseInt(ctx.Req.Header.Get(artifactXTfsFileLengthHeader), 10, 64)
226
+
if xTfsLength > 0 {
227
+
return xTfsLength, contentLength, nil
228
+
}
229
+
return contentLength, contentLength, nil
230
+
}
231
+
232
+
func (ar artifactRoutes) saveUploadChunk(ctx *context.Context,
233
+
artifact *actions.ActionArtifact,
234
+
contentSize, runID int64,
235
+
) (int64, error) {
236
+
contentRange := ctx.Req.Header.Get("Content-Range")
237
+
start, end, length := int64(0), int64(0), int64(0)
238
+
if _, err := fmt.Sscanf(contentRange, "bytes %d-%d/%d", &start, &end, &length); err != nil {
239
+
return -1, fmt.Errorf("parse content range error: %v", err)
240
+
}
241
+
242
+
storagePath := fmt.Sprintf("tmp%d/%d-%d-%d.chunk", runID, artifact.ID, start, end)
243
+
244
+
// use io.TeeReader to avoid reading all body to md5 sum.
245
+
// it writes data to hasher after reading end
246
+
// if hash is not matched, delete the read-end result
247
+
hasher := md5.New()
248
+
r := io.TeeReader(ctx.Req.Body, hasher)
249
+
250
+
// save chunk to storage
251
+
writtenSize, err := ar.fs.Save(storagePath, r, -1)
252
+
if err != nil {
253
+
return -1, fmt.Errorf("save chunk to storage error: %v", err)
254
+
}
255
+
256
+
// check md5
257
+
reqMd5String := ctx.Req.Header.Get(artifactXActionsResultsMD5Header)
258
+
chunkMd5String := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
259
+
log.Debug("[artifact] check chunk md5, sum: %s, header: %s", chunkMd5String, reqMd5String)
260
+
if reqMd5String != chunkMd5String || writtenSize != contentSize {
261
+
if err := ar.fs.Delete(storagePath); err != nil {
262
+
log.Error("Error deleting chunk: %s, %v", storagePath, err)
263
+
}
264
+
return -1, fmt.Errorf("md5 not match")
265
+
}
266
+
267
+
log.Debug("[artifact] save chunk %s, size: %d, artifact id: %d, start: %d, end: %d",
268
+
storagePath, contentSize, artifact.ID, start, end)
269
+
270
+
return length, nil
271
+
}
272
+
273
+
// The rules are from https://github.com/actions/toolkit/blob/main/packages/artifact/src/internal/path-and-artifact-name-validation.ts#L32
274
+
var invalidArtifactNameChars = strings.Join([]string{"\\", "/", "\"", ":", "<", ">", "|", "*", "?", "\r", "\n"}, "")
275
+
276
+
func (ar artifactRoutes) uploadArtifact(ctx *context.Context) {
277
+
_, runID, ok := ar.validateRunID(ctx)
278
+
if !ok {
279
+
return
280
+
}
281
+
artifactID := ctx.ParamsInt64("artifact_id")
282
+
283
+
artifact, err := actions.GetArtifactByID(ctx, artifactID)
284
+
if errors.Is(err, util.ErrNotExist) {
285
+
log.Error("Error getting artifact: %v", err)
286
+
ctx.Error(http.StatusNotFound, err.Error())
287
+
return
288
+
} else if err != nil {
289
+
log.Error("Error getting artifact: %v", err)
290
+
ctx.Error(http.StatusInternalServerError, err.Error())
291
+
return
292
+
}
293
+
294
+
// itemPath is generated from upload-artifact action
295
+
// it's formatted as {artifact_name}/{artfict_path_in_runner}
296
+
itemPath := util.PathJoinRel(ctx.Req.URL.Query().Get("itemPath"))
297
+
artifactName := strings.Split(itemPath, "/")[0]
298
+
299
+
// checkArtifactName checks if the artifact name contains invalid characters.
300
+
// If the name contains invalid characters, an error is returned.
301
+
if strings.ContainsAny(artifactName, invalidArtifactNameChars) {
302
+
log.Error("Error checking artifact name contains invalid character")
303
+
ctx.Error(http.StatusBadRequest, err.Error())
304
+
return
305
+
}
306
+
307
+
// get upload file size
308
+
fileSize, contentLength, err := ar.getUploadFileSize(ctx)
309
+
if err != nil {
310
+
log.Error("Error getting upload file size: %v", err)
311
+
ctx.Error(http.StatusInternalServerError, err.Error())
312
+
return
313
+
}
314
+
315
+
// save chunk
316
+
chunkAllLength, err := ar.saveUploadChunk(ctx, artifact, contentLength, runID)
317
+
if err != nil {
318
+
log.Error("Error saving upload chunk: %v", err)
319
+
ctx.Error(http.StatusInternalServerError, err.Error())
320
+
return
321
+
}
322
+
323
+
// if artifact name is not set, update it
324
+
if artifact.ArtifactName == "" {
325
+
artifact.ArtifactName = artifactName
326
+
artifact.ArtifactPath = itemPath // path in container
327
+
artifact.FileSize = fileSize // this is total size of all chunks
328
+
artifact.FileCompressedSize = chunkAllLength
329
+
artifact.ContentEncoding = ctx.Req.Header.Get("Content-Encoding")
330
+
if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
331
+
log.Error("Error updating artifact: %v", err)
332
+
ctx.Error(http.StatusInternalServerError, err.Error())
333
+
return
334
+
}
335
+
}
336
+
337
+
ctx.JSON(http.StatusOK, map[string]string{
338
+
"message": "success",
339
+
})
340
+
}
341
+
342
+
// comfirmUploadArtifact comfirm upload artifact.
343
+
// if all chunks are uploaded, merge them to one file.
344
+
func (ar artifactRoutes) comfirmUploadArtifact(ctx *context.Context) {
345
+
_, runID, ok := ar.validateRunID(ctx)
346
+
if !ok {
347
+
return
348
+
}
349
+
if err := ar.mergeArtifactChunks(ctx, runID); err != nil {
350
+
log.Error("Error merging chunks: %v", err)
351
+
ctx.Error(http.StatusInternalServerError, err.Error())
352
+
return
353
+
}
354
+
355
+
ctx.JSON(http.StatusOK, map[string]string{
356
+
"message": "success",
357
+
})
358
+
}
359
+
360
+
type chunkItem struct {
361
+
ArtifactID int64
362
+
Start int64
363
+
End int64
364
+
Path string
365
+
}
366
+
367
+
func (ar artifactRoutes) mergeArtifactChunks(ctx *context.Context, runID int64) error {
368
+
storageDir := fmt.Sprintf("tmp%d", runID)
369
+
var chunks []*chunkItem
370
+
if err := ar.fs.IterateObjects(storageDir, func(path string, obj storage.Object) error {
371
+
item := chunkItem{Path: path}
372
+
if _, err := fmt.Sscanf(path, storageDir+"/%d-%d-%d.chunk", &item.ArtifactID, &item.Start, &item.End); err != nil {
373
+
return fmt.Errorf("parse content range error: %v", err)
374
+
}
375
+
chunks = append(chunks, &item)
376
+
return nil
377
+
}); err != nil {
378
+
return err
379
+
}
380
+
// group chunks by artifact id
381
+
chunksMap := make(map[int64][]*chunkItem)
382
+
for _, c := range chunks {
383
+
chunksMap[c.ArtifactID] = append(chunksMap[c.ArtifactID], c)
384
+
}
385
+
386
+
for artifactID, cs := range chunksMap {
387
+
// get artifact to handle merged chunks
388
+
artifact, err := actions.GetArtifactByID(ctx, cs[0].ArtifactID)
389
+
if err != nil {
390
+
return fmt.Errorf("get artifact error: %v", err)
391
+
}
392
+
393
+
sort.Slice(cs, func(i, j int) bool {
394
+
return cs[i].Start < cs[j].Start
395
+
})
396
+
397
+
allChunks := make([]*chunkItem, 0)
398
+
startAt := int64(-1)
399
+
// check if all chunks are uploaded and in order and clean repeated chunks
400
+
for _, c := range cs {
401
+
// startAt is -1 means this is the first chunk
402
+
// previous c.ChunkEnd + 1 == c.ChunkStart means this chunk is in order
403
+
// StartAt is not -1 and c.ChunkStart is not startAt + 1 means there is a chunk missing
404
+
if c.Start == (startAt + 1) {
405
+
allChunks = append(allChunks, c)
406
+
startAt = c.End
407
+
}
408
+
}
409
+
410
+
// if the last chunk.End + 1 is not equal to chunk.ChunkLength, means chunks are not uploaded completely
411
+
if startAt+1 != artifact.FileCompressedSize {
412
+
log.Debug("[artifact] chunks are not uploaded completely, artifact_id: %d", artifactID)
413
+
break
414
+
}
415
+
416
+
// use multiReader
417
+
readers := make([]io.Reader, 0, len(allChunks))
418
+
readerClosers := make([]io.Closer, 0, len(allChunks))
419
+
for _, c := range allChunks {
420
+
reader, err := ar.fs.Open(c.Path)
421
+
if err != nil {
422
+
return fmt.Errorf("open chunk error: %v, %s", err, c.Path)
423
+
}
424
+
readers = append(readers, reader)
425
+
readerClosers = append(readerClosers, reader)
426
+
}
427
+
mergedReader := io.MultiReader(readers...)
428
+
429
+
// if chunk is gzip, decompress it
430
+
if artifact.ContentEncoding == "gzip" {
431
+
var err error
432
+
mergedReader, err = gzip.NewReader(mergedReader)
433
+
if err != nil {
434
+
return fmt.Errorf("gzip reader error: %v", err)
435
+
}
436
+
}
437
+
438
+
// save merged file
439
+
storagePath := fmt.Sprintf("%d/%d/%d.chunk", runID%255, artifactID%255, time.Now().UnixNano())
440
+
written, err := ar.fs.Save(storagePath, mergedReader, -1)
441
+
if err != nil {
442
+
return fmt.Errorf("save merged file error: %v", err)
443
+
}
444
+
if written != artifact.FileSize {
445
+
return fmt.Errorf("merged file size is not equal to chunk length")
446
+
}
447
+
448
+
// close readers
449
+
for _, r := range readerClosers {
450
+
r.Close()
451
+
}
452
+
453
+
// save storage path to artifact
454
+
log.Debug("[artifact] merge chunks to artifact: %d, %s", artifact.ID, storagePath)
455
+
artifact.StoragePath = storagePath
456
+
artifact.Status = actions.ArtifactStatusUploadConfirmed
457
+
if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
458
+
return fmt.Errorf("update artifact error: %v", err)
459
+
}
460
+
461
+
// drop chunks
462
+
for _, c := range cs {
463
+
if err := ar.fs.Delete(c.Path); err != nil {
464
+
return fmt.Errorf("delete chunk file error: %v", err)
465
+
}
466
+
}
467
+
}
468
+
return nil
469
+
}
470
+
471
+
type (
472
+
listArtifactsResponse struct {
473
+
Count int64 `json:"count"`
474
+
Value []listArtifactsResponseItem `json:"value"`
475
+
}
476
+
listArtifactsResponseItem struct {
477
+
Name string `json:"name"`
478
+
FileContainerResourceURL string `json:"fileContainerResourceUrl"`
479
+
}
480
+
)
481
+
482
+
func (ar artifactRoutes) listArtifacts(ctx *context.Context) {
483
+
_, runID, ok := ar.validateRunID(ctx)
484
+
if !ok {
485
+
return
486
+
}
487
+
488
+
artficats, err := actions.ListArtifactsByRunID(ctx, runID)
489
+
if err != nil {
490
+
log.Error("Error getting artifacts: %v", err)
491
+
ctx.Error(http.StatusInternalServerError, err.Error())
492
+
return
493
+
}
494
+
495
+
artficatsData := make([]listArtifactsResponseItem, 0, len(artficats))
496
+
for _, a := range artficats {
497
+
artficatsData = append(artficatsData, listArtifactsResponseItem{
498
+
Name: a.ArtifactName,
499
+
FileContainerResourceURL: ar.buildArtifactURL(runID, a.ID, "path"),
500
+
})
501
+
}
502
+
respData := listArtifactsResponse{
503
+
Count: int64(len(artficatsData)),
504
+
Value: artficatsData,
505
+
}
506
+
ctx.JSON(http.StatusOK, respData)
507
+
}
508
+
509
+
type (
510
+
downloadArtifactResponse struct {
511
+
Value []downloadArtifactResponseItem `json:"value"`
512
+
}
513
+
downloadArtifactResponseItem struct {
514
+
Path string `json:"path"`
515
+
ItemType string `json:"itemType"`
516
+
ContentLocation string `json:"contentLocation"`
517
+
}
518
+
)
519
+
520
+
func (ar artifactRoutes) getDownloadArtifactURL(ctx *context.Context) {
521
+
_, runID, ok := ar.validateRunID(ctx)
522
+
if !ok {
523
+
return
524
+
}
525
+
526
+
artifactID := ctx.ParamsInt64("artifact_id")
527
+
artifact, err := actions.GetArtifactByID(ctx, artifactID)
528
+
if errors.Is(err, util.ErrNotExist) {
529
+
log.Error("Error getting artifact: %v", err)
530
+
ctx.Error(http.StatusNotFound, err.Error())
531
+
return
532
+
} else if err != nil {
533
+
log.Error("Error getting artifact: %v", err)
534
+
ctx.Error(http.StatusInternalServerError, err.Error())
535
+
return
536
+
}
537
+
downloadURL := ar.buildArtifactURL(runID, artifact.ID, "download")
538
+
itemPath := util.PathJoinRel(ctx.Req.URL.Query().Get("itemPath"))
539
+
respData := downloadArtifactResponse{
540
+
Value: []downloadArtifactResponseItem{{
541
+
Path: util.PathJoinRel(itemPath, artifact.ArtifactPath),
542
+
ItemType: "file",
543
+
ContentLocation: downloadURL,
544
+
}},
545
+
}
546
+
ctx.JSON(http.StatusOK, respData)
547
+
}
548
+
549
+
func (ar artifactRoutes) downloadArtifact(ctx *context.Context) {
550
+
_, runID, ok := ar.validateRunID(ctx)
551
+
if !ok {
552
+
return
553
+
}
554
+
555
+
artifactID := ctx.ParamsInt64("artifact_id")
556
+
artifact, err := actions.GetArtifactByID(ctx, artifactID)
557
+
if errors.Is(err, util.ErrNotExist) {
558
+
log.Error("Error getting artifact: %v", err)
559
+
ctx.Error(http.StatusNotFound, err.Error())
560
+
return
561
+
} else if err != nil {
562
+
log.Error("Error getting artifact: %v", err)
563
+
ctx.Error(http.StatusInternalServerError, err.Error())
564
+
return
565
+
}
566
+
if artifact.RunID != runID {
567
+
log.Error("Error dismatch runID and artifactID, task: %v, artifact: %v", runID, artifactID)
568
+
ctx.Error(http.StatusBadRequest, err.Error())
569
+
return
570
+
}
571
+
572
+
fd, err := ar.fs.Open(artifact.StoragePath)
573
+
if err != nil {
574
+
log.Error("Error opening file: %v", err)
575
+
ctx.Error(http.StatusInternalServerError, err.Error())
576
+
return
577
+
}
578
+
defer fd.Close()
579
+
580
+
if strings.HasSuffix(artifact.ArtifactPath, ".gz") {
581
+
ctx.Resp.Header().Set("Content-Encoding", "gzip")
582
+
}
583
+
ctx.ServeContent(fd, &context.ServeHeaderOptions{
584
+
Filename: artifact.ArtifactName,
585
+
LastModified: artifact.CreatedUnix.AsLocalTime(),
586
+
})
587
+
}
+6
routers/init.go
+6
routers/init.go
···
193
193
if setting.Actions.Enabled {
194
194
prefix := "/api/actions"
195
195
r.Mount(prefix, actions_router.Routes(ctx, prefix))
196
+
197
+
// TODO: Pipeline api used for runner internal communication with gitea server. but only artifact is used for now.
198
+
// In Github, it uses ACTIONS_RUNTIME_URL=https://pipelines.actions.githubusercontent.com/fLgcSHkPGySXeIFrg8W8OBSfeg3b5Fls1A1CwX566g8PayEGlg/
199
+
// TODO: this prefix should be generated with a token string with runner ?
200
+
prefix = "/api/actions_pipeline"
201
+
r.Mount(prefix, actions_router.ArtifactsRoutes(ctx, prefix))
196
202
}
197
203
198
204
return r
+78
routers/web/repo/actions/view.go
+78
routers/web/repo/actions/view.go
···
16
16
"code.gitea.io/gitea/modules/actions"
17
17
"code.gitea.io/gitea/modules/base"
18
18
context_module "code.gitea.io/gitea/modules/context"
19
+
"code.gitea.io/gitea/modules/storage"
19
20
"code.gitea.io/gitea/modules/timeutil"
20
21
"code.gitea.io/gitea/modules/util"
21
22
"code.gitea.io/gitea/modules/web"
···
418
419
}
419
420
return jobs[0], jobs
420
421
}
422
+
423
+
type ArtifactsViewResponse struct {
424
+
Artifacts []*ArtifactsViewItem `json:"artifacts"`
425
+
}
426
+
427
+
type ArtifactsViewItem struct {
428
+
Name string `json:"name"`
429
+
Size int64 `json:"size"`
430
+
ID int64 `json:"id"`
431
+
}
432
+
433
+
func ArtifactsView(ctx *context_module.Context) {
434
+
runIndex := ctx.ParamsInt64("run")
435
+
run, err := actions_model.GetRunByIndex(ctx, ctx.Repo.Repository.ID, runIndex)
436
+
if err != nil {
437
+
if errors.Is(err, util.ErrNotExist) {
438
+
ctx.Error(http.StatusNotFound, err.Error())
439
+
return
440
+
}
441
+
ctx.Error(http.StatusInternalServerError, err.Error())
442
+
return
443
+
}
444
+
artifacts, err := actions_model.ListUploadedArtifactsByRunID(ctx, run.ID)
445
+
if err != nil {
446
+
ctx.Error(http.StatusInternalServerError, err.Error())
447
+
return
448
+
}
449
+
artifactsResponse := ArtifactsViewResponse{
450
+
Artifacts: make([]*ArtifactsViewItem, 0, len(artifacts)),
451
+
}
452
+
for _, art := range artifacts {
453
+
artifactsResponse.Artifacts = append(artifactsResponse.Artifacts, &ArtifactsViewItem{
454
+
Name: art.ArtifactName,
455
+
Size: art.FileSize,
456
+
ID: art.ID,
457
+
})
458
+
}
459
+
ctx.JSON(http.StatusOK, artifactsResponse)
460
+
}
461
+
462
+
func ArtifactsDownloadView(ctx *context_module.Context) {
463
+
runIndex := ctx.ParamsInt64("run")
464
+
artifactID := ctx.ParamsInt64("id")
465
+
466
+
artifact, err := actions_model.GetArtifactByID(ctx, artifactID)
467
+
if errors.Is(err, util.ErrNotExist) {
468
+
ctx.Error(http.StatusNotFound, err.Error())
469
+
} else if err != nil {
470
+
ctx.Error(http.StatusInternalServerError, err.Error())
471
+
return
472
+
}
473
+
run, err := actions_model.GetRunByIndex(ctx, ctx.Repo.Repository.ID, runIndex)
474
+
if err != nil {
475
+
if errors.Is(err, util.ErrNotExist) {
476
+
ctx.Error(http.StatusNotFound, err.Error())
477
+
return
478
+
}
479
+
ctx.Error(http.StatusInternalServerError, err.Error())
480
+
return
481
+
}
482
+
if artifact.RunID != run.ID {
483
+
ctx.Error(http.StatusNotFound, "artifact not found")
484
+
return
485
+
}
486
+
487
+
f, err := storage.ActionsArtifacts.Open(artifact.StoragePath)
488
+
if err != nil {
489
+
ctx.Error(http.StatusInternalServerError, err.Error())
490
+
return
491
+
}
492
+
defer f.Close()
493
+
494
+
ctx.ServeContent(f, &context_module.ServeHeaderOptions{
495
+
Filename: artifact.ArtifactName,
496
+
LastModified: artifact.CreatedUnix.AsLocalTime(),
497
+
})
498
+
}
+2
routers/web/web.go
+2
routers/web/web.go
···
1192
1192
})
1193
1193
m.Post("/cancel", reqRepoActionsWriter, actions.Cancel)
1194
1194
m.Post("/approve", reqRepoActionsWriter, actions.Approve)
1195
+
m.Post("/artifacts", actions.ArtifactsView)
1196
+
m.Get("/artifacts/{id}", actions.ArtifactsDownloadView)
1195
1197
m.Post("/rerun", reqRepoActionsWriter, actions.RerunAll)
1196
1198
})
1197
1199
}, reqRepoActionsReader, actions.MustEnableActions)
+1
templates/repo/actions/view.tmpl
+1
templates/repo/actions/view.tmpl
···
17
17
data-locale-status-cancelled="{{.locale.Tr "actions.status.cancelled"}}"
18
18
data-locale-status-skipped="{{.locale.Tr "actions.status.skipped"}}"
19
19
data-locale-status-blocked="{{.locale.Tr "actions.status.blocked"}}"
20
+
data-locale-artifacts-title="{{$.locale.Tr "artifacts"}}"
20
21
>
21
22
</div>
22
23
</div>
+143
tests/integration/api_actions_artifact_test.go
+143
tests/integration/api_actions_artifact_test.go
···
1
+
// Copyright 2023 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package integration
5
+
6
+
import (
7
+
"net/http"
8
+
"strings"
9
+
"testing"
10
+
11
+
"code.gitea.io/gitea/tests"
12
+
13
+
"github.com/stretchr/testify/assert"
14
+
)
15
+
16
+
func TestActionsArtifactUpload(t *testing.T) {
17
+
defer tests.PrepareTestEnv(t)()
18
+
19
+
type uploadArtifactResponse struct {
20
+
FileContainerResourceURL string `json:"fileContainerResourceUrl"`
21
+
}
22
+
23
+
type getUploadArtifactRequest struct {
24
+
Type string
25
+
Name string
26
+
}
27
+
28
+
// acquire artifact upload url
29
+
req := NewRequestWithJSON(t, "POST", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts", getUploadArtifactRequest{
30
+
Type: "actions_storage",
31
+
Name: "artifact",
32
+
})
33
+
req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
34
+
resp := MakeRequest(t, req, http.StatusOK)
35
+
var uploadResp uploadArtifactResponse
36
+
DecodeJSON(t, resp, &uploadResp)
37
+
assert.Contains(t, uploadResp.FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts")
38
+
39
+
// get upload url
40
+
idx := strings.Index(uploadResp.FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/")
41
+
url := uploadResp.FileContainerResourceURL[idx:] + "?itemPath=artifact/abc.txt"
42
+
43
+
// upload artifact chunk
44
+
body := strings.Repeat("A", 1024)
45
+
req = NewRequestWithBody(t, "PUT", url, strings.NewReader(body))
46
+
req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
47
+
req.Header.Add("Content-Range", "bytes 0-1023/1024")
48
+
req.Header.Add("x-tfs-filelength", "1024")
49
+
req.Header.Add("x-actions-results-md5", "1HsSe8LeLWh93ILaw1TEFQ==") // base64(md5(body))
50
+
MakeRequest(t, req, http.StatusOK)
51
+
52
+
t.Logf("Create artifact confirm")
53
+
54
+
// confirm artifact upload
55
+
req = NewRequest(t, "PATCH", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts")
56
+
req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
57
+
MakeRequest(t, req, http.StatusOK)
58
+
}
59
+
60
+
func TestActionsArtifactUploadNotExist(t *testing.T) {
61
+
defer tests.PrepareTestEnv(t)()
62
+
63
+
// artifact id 54321 not exist
64
+
url := "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts/54321/upload?itemPath=artifact/abc.txt"
65
+
body := strings.Repeat("A", 1024)
66
+
req := NewRequestWithBody(t, "PUT", url, strings.NewReader(body))
67
+
req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
68
+
req.Header.Add("Content-Range", "bytes 0-1023/1024")
69
+
req.Header.Add("x-tfs-filelength", "1024")
70
+
req.Header.Add("x-actions-results-md5", "1HsSe8LeLWh93ILaw1TEFQ==") // base64(md5(body))
71
+
MakeRequest(t, req, http.StatusNotFound)
72
+
}
73
+
74
+
func TestActionsArtifactConfirmUpload(t *testing.T) {
75
+
defer tests.PrepareTestEnv(t)()
76
+
77
+
req := NewRequest(t, "PATCH", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts")
78
+
req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
79
+
resp := MakeRequest(t, req, http.StatusOK)
80
+
assert.Contains(t, resp.Body.String(), "success")
81
+
}
82
+
83
+
func TestActionsArtifactUploadWithoutToken(t *testing.T) {
84
+
defer tests.PrepareTestEnv(t)()
85
+
86
+
req := NewRequestWithJSON(t, "POST", "/api/actions_pipeline/_apis/pipelines/workflows/1/artifacts", nil)
87
+
MakeRequest(t, req, http.StatusUnauthorized)
88
+
}
89
+
90
+
func TestActionsArtifactDownload(t *testing.T) {
91
+
defer tests.PrepareTestEnv(t)()
92
+
93
+
type (
94
+
listArtifactsResponseItem struct {
95
+
Name string `json:"name"`
96
+
FileContainerResourceURL string `json:"fileContainerResourceUrl"`
97
+
}
98
+
listArtifactsResponse struct {
99
+
Count int64 `json:"count"`
100
+
Value []listArtifactsResponseItem `json:"value"`
101
+
}
102
+
)
103
+
104
+
req := NewRequest(t, "GET", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts")
105
+
req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
106
+
resp := MakeRequest(t, req, http.StatusOK)
107
+
var listResp listArtifactsResponse
108
+
DecodeJSON(t, resp, &listResp)
109
+
assert.Equal(t, int64(1), listResp.Count)
110
+
assert.Equal(t, "artifact", listResp.Value[0].Name)
111
+
assert.Contains(t, listResp.Value[0].FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts")
112
+
113
+
type (
114
+
downloadArtifactResponseItem struct {
115
+
Path string `json:"path"`
116
+
ItemType string `json:"itemType"`
117
+
ContentLocation string `json:"contentLocation"`
118
+
}
119
+
downloadArtifactResponse struct {
120
+
Value []downloadArtifactResponseItem `json:"value"`
121
+
}
122
+
)
123
+
124
+
idx := strings.Index(listResp.Value[0].FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/")
125
+
url := listResp.Value[0].FileContainerResourceURL[idx+1:]
126
+
req = NewRequest(t, "GET", url)
127
+
req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
128
+
resp = MakeRequest(t, req, http.StatusOK)
129
+
var downloadResp downloadArtifactResponse
130
+
DecodeJSON(t, resp, &downloadResp)
131
+
assert.Len(t, downloadResp.Value, 1)
132
+
assert.Equal(t, "artifact/abc.txt", downloadResp.Value[0].Path)
133
+
assert.Equal(t, "file", downloadResp.Value[0].ItemType)
134
+
assert.Contains(t, downloadResp.Value[0].ContentLocation, "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts")
135
+
136
+
idx = strings.Index(downloadResp.Value[0].ContentLocation, "/api/actions_pipeline/_apis/pipelines/")
137
+
url = downloadResp.Value[0].ContentLocation[idx:]
138
+
req = NewRequest(t, "GET", url)
139
+
req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a")
140
+
resp = MakeRequest(t, req, http.StatusOK)
141
+
body := strings.Repeat("A", 1024)
142
+
assert.Equal(t, resp.Body.String(), body)
143
+
}
+3
tests/mssql.ini.tmpl
+3
tests/mssql.ini.tmpl
+3
tests/mysql.ini.tmpl
+3
tests/mysql.ini.tmpl
+3
tests/mysql8.ini.tmpl
+3
tests/mysql8.ini.tmpl
+3
tests/pgsql.ini.tmpl
+3
tests/pgsql.ini.tmpl
+3
tests/sqlite.ini.tmpl
+3
tests/sqlite.ini.tmpl
+50
web_src/js/components/RepoActionView.vue
+50
web_src/js/components/RepoActionView.vue
···
42
42
</div>
43
43
</div>
44
44
</div>
45
+
<div class="job-artifacts" v-if="artifacts.length > 0">
46
+
<div class="job-artifacts-title">
47
+
{{ locale.artifactsTitle }}
48
+
</div>
49
+
<ul class="job-artifacts-list">
50
+
<li class="job-artifacts-item" v-for="artifact in artifacts" :key="artifact.id">
51
+
<a class="job-artifacts-link" target="_blank" :href="run.link+'/artifacts/'+artifact.id">
52
+
<SvgIcon name="octicon-file" class="ui text black job-artifacts-icon" />{{ artifact.name }}
53
+
</a>
54
+
</li>
55
+
</ul>
56
+
</div>
45
57
</div>
46
58
47
59
<div class="action-view-right">
···
102
114
loading: false,
103
115
intervalID: null,
104
116
currentJobStepsStates: [],
117
+
artifacts: [],
105
118
106
119
// provided by backend
107
120
run: {
···
156
169
this.intervalID = setInterval(this.loadJob, 1000);
157
170
},
158
171
172
+
unmounted() {
173
+
// clear the interval timer when the component is unmounted
174
+
// even our page is rendered once, not spa style
175
+
if (this.intervalID) {
176
+
clearInterval(this.intervalID);
177
+
this.intervalID = null;
178
+
}
179
+
},
180
+
159
181
methods: {
160
182
// get the active container element, either the `job-step-logs` or the `job-log-list` in the `job-log-group`
161
183
getLogsContainer(idx) {
···
259
281
try {
260
282
this.loading = true;
261
283
284
+
// refresh artifacts if upload-artifact step done
285
+
const resp = await this.fetchPost(`${this.actionsURL}/runs/${this.runIndex}/artifacts`);
286
+
const artifacts = await resp.json();
287
+
this.artifacts = artifacts['artifacts'] || [];
288
+
262
289
const response = await this.fetchJob();
263
290
264
291
// save the state to Vue data, then the UI will be updated
···
286
313
this.loading = false;
287
314
}
288
315
},
316
+
289
317
290
318
fetchPost(url, body) {
291
319
return fetch(url, {
···
319
347
approve: el.getAttribute('data-locale-approve'),
320
348
cancel: el.getAttribute('data-locale-cancel'),
321
349
rerun: el.getAttribute('data-locale-rerun'),
350
+
artifactsTitle: el.getAttribute('data-locale-artifacts-title'),
322
351
status: {
323
352
unknown: el.getAttribute('data-locale-status-unknown'),
324
353
waiting: el.getAttribute('data-locale-status-waiting'),
···
421
450
.job-group-section .job-group-summary {
422
451
margin: 5px 0;
423
452
padding: 10px;
453
+
}
454
+
455
+
.job-artifacts-title {
456
+
font-size: 18px;
457
+
margin-top: 16px;
458
+
padding: 16px 10px 0px 20px;
459
+
border-top: 1px solid var(--color-secondary);
460
+
}
461
+
462
+
.job-artifacts-item {
463
+
margin: 5px 0;
464
+
padding: 6px;
465
+
}
466
+
467
+
.job-artifacts-list {
468
+
padding-left: 12px;
469
+
list-style: none;
470
+
}
471
+
472
+
.job-artifacts-icon {
473
+
padding-right: 3px;
424
474
}
425
475
426
476
.job-group-section .job-brief-list .job-brief-item {