Actions Artifacts support uploading multiple files and directories (#24874)

+44 -15

models/actions/artifact.go

··· 31 31 // ActionArtifact is a file that is stored in the artifact storage. 32 32 type ActionArtifact struct { 33 33 ID int64 `xorm:"pk autoincr"` 34 - RunID int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact 34 + RunID int64 `xorm:"index unique(runid_name_path)"` // The run id of the artifact 35 35 RunnerID int64 36 36 RepoID int64 `xorm:"index"` 37 37 OwnerID int64 ··· 40 40 FileSize int64 // The size of the artifact in bytes 41 41 FileCompressedSize int64 // The size of the artifact in bytes after gzip compression 42 42 ContentEncoding string // The content encoding of the artifact 43 - ArtifactPath string // The path to the artifact when runner uploads it 44 - ArtifactName string `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it 45 - Status int64 `xorm:"index"` // The status of the artifact, uploading, expired or need-delete 43 + ArtifactPath string `xorm:"index unique(runid_name_path)"` // The path to the artifact when runner uploads it 44 + ArtifactName string `xorm:"index unique(runid_name_path)"` // The name of the artifact when runner uploads it 45 + Status int64 `xorm:"index"` // The status of the artifact, uploading, expired or need-delete 46 46 CreatedUnix timeutil.TimeStamp `xorm:"created"` 47 47 UpdatedUnix timeutil.TimeStamp `xorm:"updated index"` 48 48 } 49 49 50 - // CreateArtifact create a new artifact with task info or get same named artifact in the same run 51 - func CreateArtifact(ctx context.Context, t *ActionTask, artifactName string) (*ActionArtifact, error) { 50 + func CreateArtifact(ctx context.Context, t *ActionTask, artifactName, artifactPath string) (*ActionArtifact, error) { 52 51 if err := t.LoadJob(ctx); err != nil { 53 52 return nil, err 54 53 } 55 - artifact, err := getArtifactByArtifactName(ctx, t.Job.RunID, artifactName) 54 + artifact, err := getArtifactByNameAndPath(ctx, t.Job.RunID, artifactName, artifactPath) 56 55 if errors.Is(err, util.ErrNotExist) { 57 56 artifact := &ActionArtifact{ 58 - RunID: t.Job.RunID, 59 - RunnerID: t.RunnerID, 60 - RepoID: t.RepoID, 61 - OwnerID: t.OwnerID, 62 - CommitSHA: t.CommitSHA, 63 - Status: ArtifactStatusUploadPending, 57 + ArtifactName: artifactName, 58 + ArtifactPath: artifactPath, 59 + RunID: t.Job.RunID, 60 + RunnerID: t.RunnerID, 61 + RepoID: t.RepoID, 62 + OwnerID: t.OwnerID, 63 + CommitSHA: t.CommitSHA, 64 + Status: ArtifactStatusUploadPending, 64 65 } 65 66 if _, err := db.GetEngine(ctx).Insert(artifact); err != nil { 66 67 return nil, err ··· 72 73 return artifact, nil 73 74 } 74 75 75 - func getArtifactByArtifactName(ctx context.Context, runID int64, name string) (*ActionArtifact, error) { 76 + func getArtifactByNameAndPath(ctx context.Context, runID int64, name, fpath string) (*ActionArtifact, error) { 76 77 var art ActionArtifact 77 - has, err := db.GetEngine(ctx).Where("run_id = ? AND artifact_name = ?", runID, name).Get(&art) 78 + has, err := db.GetEngine(ctx).Where("run_id = ? AND artifact_name = ? AND artifact_path = ?", runID, name, fpath).Get(&art) 78 79 if err != nil { 79 80 return nil, err 80 81 } else if !has { ··· 109 110 return arts, db.GetEngine(ctx).Where("run_id=?", runID).Find(&arts) 110 111 } 111 112 113 + // ListArtifactsByRunIDAndArtifactName returns an artifacts of a run by artifact name 114 + func ListArtifactsByRunIDAndArtifactName(ctx context.Context, runID int64, artifactName string) ([]*ActionArtifact, error) { 115 + arts := make([]*ActionArtifact, 0, 10) 116 + return arts, db.GetEngine(ctx).Where("run_id=? AND artifact_name=?", runID, artifactName).Find(&arts) 117 + } 118 + 112 119 // ListUploadedArtifactsByRunID returns all uploaded artifacts of a run 113 120 func ListUploadedArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) { 114 121 arts := make([]*ActionArtifact, 0, 10) 115 122 return arts, db.GetEngine(ctx).Where("run_id=? AND status=?", runID, ArtifactStatusUploadConfirmed).Find(&arts) 116 123 } 117 124 125 + // ActionArtifactMeta is the meta data of an artifact 126 + type ActionArtifactMeta struct { 127 + ArtifactName string 128 + FileSize int64 129 + } 130 + 131 + // ListUploadedArtifactsMeta returns all uploaded artifacts meta of a run 132 + func ListUploadedArtifactsMeta(ctx context.Context, runID int64) ([]*ActionArtifactMeta, error) { 133 + arts := make([]*ActionArtifactMeta, 0, 10) 134 + return arts, db.GetEngine(ctx).Table("action_artifact"). 135 + Where("run_id=? AND status=?", runID, ArtifactStatusUploadConfirmed). 136 + GroupBy("artifact_name"). 137 + Select("artifact_name, sum(file_size) as file_size"). 138 + Find(&arts) 139 + } 140 + 118 141 // ListArtifactsByRepoID returns all artifacts of a repo 119 142 func ListArtifactsByRepoID(ctx context.Context, repoID int64) ([]*ActionArtifact, error) { 120 143 arts := make([]*ActionArtifact, 0, 10) 121 144 return arts, db.GetEngine(ctx).Where("repo_id=?", repoID).Find(&arts) 122 145 } 146 + 147 + // ListArtifactsByRunIDAndName returns artifacts by name of a run 148 + func ListArtifactsByRunIDAndName(ctx context.Context, runID int64, name string) ([]*ActionArtifact, error) { 149 + arts := make([]*ActionArtifact, 0, 10) 150 + return arts, db.GetEngine(ctx).Where("run_id=? AND artifact_name=?", runID, name).Find(&arts) 151 + }

+2

models/migrations/migrations.go

··· 511 511 NewMigration("Add git_size and lfs_size columns to repository table", v1_21.AddGitSizeAndLFSSizeToRepositoryTable), 512 512 // v264 -> v265 513 513 NewMigration("Add branch table", v1_21.AddBranchTable), 514 + // v265 -> v266 515 + NewMigration("Alter Actions Artifact table", v1_21.AlterActionArtifactTable), 514 516 } 515 517 516 518 // GetCurrentDBVersion returns the current db version

+19

models/migrations/v1_21/v265.go

··· 1 + // Copyright 2023 The Gitea Authors. All rights reserved. 2 + // SPDX-License-Identifier: MIT 3 + 4 + package v1_21 //nolint 5 + 6 + import ( 7 + "xorm.io/xorm" 8 + ) 9 + 10 + func AlterActionArtifactTable(x *xorm.Engine) error { 11 + // ActionArtifact is a file that is stored in the artifact storage. 12 + type ActionArtifact struct { 13 + RunID int64 `xorm:"index unique(runid_name_path)"` // The run id of the artifact 14 + ArtifactPath string `xorm:"index unique(runid_name_path)"` // The path to the artifact when runner uploads it 15 + ArtifactName string `xorm:"index unique(runid_name_path)"` // The name of the artifact when 16 + } 17 + 18 + return x.Sync(new(ActionArtifact)) 19 + }

+108 -272

routers/api/actions/artifacts.go

··· 62 62 // 63 63 64 64 import ( 65 - "compress/gzip" 66 65 "crypto/md5" 67 - "encoding/base64" 68 66 "errors" 69 67 "fmt" 70 - "io" 71 68 "net/http" 72 - "sort" 73 69 "strconv" 74 70 "strings" 75 - "time" 76 71 77 72 "code.gitea.io/gitea/models/actions" 78 73 "code.gitea.io/gitea/modules/context" ··· 83 78 "code.gitea.io/gitea/modules/util" 84 79 "code.gitea.io/gitea/modules/web" 85 80 web_types "code.gitea.io/gitea/modules/web/types" 86 - ) 87 - 88 - const ( 89 - artifactXTfsFileLengthHeader = "x-tfs-filelength" 90 - artifactXActionsResultsMD5Header = "x-actions-results-md5" 91 81 ) 92 82 93 83 const artifactRouteBase = "/_apis/pipelines/workflows/{run_id}/artifacts" ··· 121 111 // retrieve, list and confirm artifacts 122 112 m.Combo("").Get(r.listArtifacts).Post(r.getUploadArtifactURL).Patch(r.comfirmUploadArtifact) 123 113 // handle container artifacts list and download 124 - m.Group("/{artifact_id}", func() { 125 - m.Put("/upload", r.uploadArtifact) 126 - m.Get("/path", r.getDownloadArtifactURL) 127 - m.Get("/download", r.downloadArtifact) 128 - }) 114 + m.Put("/{artifact_hash}/upload", r.uploadArtifact) 115 + // handle artifacts download 116 + m.Get("/{artifact_hash}/download_url", r.getDownloadArtifactURL) 117 + m.Get("/{artifact_id}/download", r.downloadArtifact) 129 118 }) 130 119 131 120 return m ··· 173 162 fs storage.ObjectStorage 174 163 } 175 164 176 - func (ar artifactRoutes) buildArtifactURL(runID, artifactID int64, suffix string) string { 165 + func (ar artifactRoutes) buildArtifactURL(runID int64, artifactHash, suffix string) string { 177 166 uploadURL := strings.TrimSuffix(setting.AppURL, "/") + strings.TrimSuffix(ar.prefix, "/") + 178 167 strings.ReplaceAll(artifactRouteBase, "{run_id}", strconv.FormatInt(runID, 10)) + 179 - "/" + strconv.FormatInt(artifactID, 10) + "/" + suffix 168 + "/" + artifactHash + "/" + suffix 180 169 return uploadURL 181 170 } 182 171 ··· 189 178 FileContainerResourceURL string `json:"fileContainerResourceUrl"` 190 179 } 191 180 192 - func (ar artifactRoutes) validateRunID(ctx *ArtifactContext) (*actions.ActionTask, int64, bool) { 193 - task := ctx.ActionTask 194 - runID := ctx.ParamsInt64("run_id") 195 - if task.Job.RunID != runID { 196 - log.Error("Error runID not match") 197 - ctx.Error(http.StatusBadRequest, "run-id does not match") 198 - return nil, 0, false 199 - } 200 - return task, runID, true 201 - } 202 - 203 181 // getUploadArtifactURL generates a URL for uploading an artifact 204 182 func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) { 205 - task, runID, ok := ar.validateRunID(ctx) 183 + _, runID, ok := validateRunID(ctx) 206 184 if !ok { 207 185 return 208 186 } ··· 214 192 return 215 193 } 216 194 217 - artifact, err := actions.CreateArtifact(ctx, task, req.Name) 218 - if err != nil { 219 - log.Error("Error creating artifact: %v", err) 220 - ctx.Error(http.StatusInternalServerError, err.Error()) 221 - return 222 - } 195 + // use md5(artifact_name) to create upload url 196 + artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(req.Name))) 223 197 resp := getUploadArtifactResponse{ 224 - FileContainerResourceURL: ar.buildArtifactURL(runID, artifact.ID, "upload"), 198 + FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "upload"), 225 199 } 226 - log.Debug("[artifact] get upload url: %s, artifact id: %d", resp.FileContainerResourceURL, artifact.ID) 200 + log.Debug("[artifact] get upload url: %s", resp.FileContainerResourceURL) 227 201 ctx.JSON(http.StatusOK, resp) 228 202 } 229 203 230 - // getUploadFileSize returns the size of the file to be uploaded. 231 - // The raw size is the size of the file as reported by the header X-TFS-FileLength. 232 - func (ar artifactRoutes) getUploadFileSize(ctx *ArtifactContext) (int64, int64, error) { 233 - contentLength := ctx.Req.ContentLength 234 - xTfsLength, _ := strconv.ParseInt(ctx.Req.Header.Get(artifactXTfsFileLengthHeader), 10, 64) 235 - if xTfsLength > 0 { 236 - return xTfsLength, contentLength, nil 237 - } 238 - return contentLength, contentLength, nil 239 - } 240 - 241 - func (ar artifactRoutes) saveUploadChunk(ctx *ArtifactContext, 242 - artifact *actions.ActionArtifact, 243 - contentSize, runID int64, 244 - ) (int64, error) { 245 - contentRange := ctx.Req.Header.Get("Content-Range") 246 - start, end, length := int64(0), int64(0), int64(0) 247 - if _, err := fmt.Sscanf(contentRange, "bytes %d-%d/%d", &start, &end, &length); err != nil { 248 - return -1, fmt.Errorf("parse content range error: %v", err) 249 - } 250 - 251 - storagePath := fmt.Sprintf("tmp%d/%d-%d-%d.chunk", runID, artifact.ID, start, end) 252 - 253 - // use io.TeeReader to avoid reading all body to md5 sum. 254 - // it writes data to hasher after reading end 255 - // if hash is not matched, delete the read-end result 256 - hasher := md5.New() 257 - r := io.TeeReader(ctx.Req.Body, hasher) 258 - 259 - // save chunk to storage 260 - writtenSize, err := ar.fs.Save(storagePath, r, -1) 261 - if err != nil { 262 - return -1, fmt.Errorf("save chunk to storage error: %v", err) 263 - } 264 - 265 - // check md5 266 - reqMd5String := ctx.Req.Header.Get(artifactXActionsResultsMD5Header) 267 - chunkMd5String := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) 268 - log.Debug("[artifact] check chunk md5, sum: %s, header: %s", chunkMd5String, reqMd5String) 269 - if reqMd5String != chunkMd5String || writtenSize != contentSize { 270 - if err := ar.fs.Delete(storagePath); err != nil { 271 - log.Error("Error deleting chunk: %s, %v", storagePath, err) 272 - } 273 - return -1, fmt.Errorf("md5 not match") 274 - } 275 - 276 - log.Debug("[artifact] save chunk %s, size: %d, artifact id: %d, start: %d, end: %d", 277 - storagePath, contentSize, artifact.ID, start, end) 278 - 279 - return length, nil 280 - } 281 - 282 - // The rules are from https://github.com/actions/toolkit/blob/main/packages/artifact/src/internal/path-and-artifact-name-validation.ts#L32 283 - var invalidArtifactNameChars = strings.Join([]string{"\\", "/", "\"", ":", "<", ">", "|", "*", "?", "\r", "\n"}, "") 284 - 285 204 func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) { 286 - _, runID, ok := ar.validateRunID(ctx) 205 + task, runID, ok := validateRunID(ctx) 287 206 if !ok { 288 207 return 289 208 } 290 - artifactID := ctx.ParamsInt64("artifact_id") 291 - 292 - artifact, err := actions.GetArtifactByID(ctx, artifactID) 293 - if errors.Is(err, util.ErrNotExist) { 294 - log.Error("Error getting artifact: %v", err) 295 - ctx.Error(http.StatusNotFound, err.Error()) 296 - return 297 - } else if err != nil { 298 - log.Error("Error getting artifact: %v", err) 299 - ctx.Error(http.StatusInternalServerError, err.Error()) 209 + artifactName, artifactPath, ok := parseArtifactItemPath(ctx) 210 + if !ok { 300 211 return 301 212 } 302 213 303 - // itemPath is generated from upload-artifact action 304 - // it's formatted as {artifact_name}/{artfict_path_in_runner} 305 - itemPath := util.PathJoinRel(ctx.Req.URL.Query().Get("itemPath")) 306 - artifactName := strings.Split(itemPath, "/")[0] 307 - 308 - // checkArtifactName checks if the artifact name contains invalid characters. 309 - // If the name contains invalid characters, an error is returned. 310 - if strings.ContainsAny(artifactName, invalidArtifactNameChars) { 311 - log.Error("Error checking artifact name contains invalid character") 312 - ctx.Error(http.StatusBadRequest, err.Error()) 214 + // get upload file size 215 + fileRealTotalSize, contentLength, err := getUploadFileSize(ctx) 216 + if err != nil { 217 + log.Error("Error get upload file size: %v", err) 218 + ctx.Error(http.StatusInternalServerError, "Error get upload file size") 313 219 return 314 220 } 315 221 316 - // get upload file size 317 - fileSize, contentLength, err := ar.getUploadFileSize(ctx) 222 + // create or get artifact with name and path 223 + artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath) 318 224 if err != nil { 319 - log.Error("Error getting upload file size: %v", err) 320 - ctx.Error(http.StatusInternalServerError, err.Error()) 225 + log.Error("Error create or get artifact: %v", err) 226 + ctx.Error(http.StatusInternalServerError, "Error create or get artifact") 321 227 return 322 228 } 323 229 324 - // save chunk 325 - chunkAllLength, err := ar.saveUploadChunk(ctx, artifact, contentLength, runID) 230 + // save chunk to storage, if success, return chunk stotal size 231 + // if artifact is not gzip when uploading, chunksTotalSize == fileRealTotalSize 232 + // if artifact is gzip when uploading, chunksTotalSize < fileRealTotalSize 233 + chunksTotalSize, err := saveUploadChunk(ar.fs, ctx, artifact, contentLength, runID) 326 234 if err != nil { 327 - log.Error("Error saving upload chunk: %v", err) 328 - ctx.Error(http.StatusInternalServerError, err.Error()) 235 + log.Error("Error save upload chunk: %v", err) 236 + ctx.Error(http.StatusInternalServerError, "Error save upload chunk") 329 237 return 330 238 } 331 239 332 - // if artifact name is not set, update it 333 - if artifact.ArtifactName == "" { 334 - artifact.ArtifactName = artifactName 335 - artifact.ArtifactPath = itemPath // path in container 336 - artifact.FileSize = fileSize // this is total size of all chunks 337 - artifact.FileCompressedSize = chunkAllLength 240 + // update artifact size if zero 241 + if artifact.FileSize == 0 || artifact.FileCompressedSize == 0 { 242 + artifact.FileSize = fileRealTotalSize 243 + artifact.FileCompressedSize = chunksTotalSize 338 244 artifact.ContentEncoding = ctx.Req.Header.Get("Content-Encoding") 339 245 if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil { 340 - log.Error("Error updating artifact: %v", err) 341 - ctx.Error(http.StatusInternalServerError, err.Error()) 246 + log.Error("Error update artifact: %v", err) 247 + ctx.Error(http.StatusInternalServerError, "Error update artifact") 342 248 return 343 249 } 344 250 } ··· 351 257 // comfirmUploadArtifact comfirm upload artifact. 352 258 // if all chunks are uploaded, merge them to one file. 353 259 func (ar artifactRoutes) comfirmUploadArtifact(ctx *ArtifactContext) { 354 - _, runID, ok := ar.validateRunID(ctx) 260 + _, runID, ok := validateRunID(ctx) 355 261 if !ok { 356 262 return 357 263 } 358 - if err := ar.mergeArtifactChunks(ctx, runID); err != nil { 359 - log.Error("Error merging chunks: %v", err) 360 - ctx.Error(http.StatusInternalServerError, err.Error()) 264 + artifactName := ctx.Req.URL.Query().Get("artifactName") 265 + if artifactName == "" { 266 + log.Error("Error artifact name is empty") 267 + ctx.Error(http.StatusBadRequest, "Error artifact name is empty") 361 268 return 362 269 } 363 - 270 + if err := mergeChunksForRun(ctx, ar.fs, runID, artifactName); err != nil { 271 + log.Error("Error merge chunks: %v", err) 272 + ctx.Error(http.StatusInternalServerError, "Error merge chunks") 273 + return 274 + } 364 275 ctx.JSON(http.StatusOK, map[string]string{ 365 276 "message": "success", 366 277 }) 367 278 } 368 279 369 - type chunkItem struct { 370 - ArtifactID int64 371 - Start int64 372 - End int64 373 - Path string 374 - } 375 - 376 - func (ar artifactRoutes) mergeArtifactChunks(ctx *ArtifactContext, runID int64) error { 377 - storageDir := fmt.Sprintf("tmp%d", runID) 378 - var chunks []*chunkItem 379 - if err := ar.fs.IterateObjects(storageDir, func(path string, obj storage.Object) error { 380 - item := chunkItem{Path: path} 381 - if _, err := fmt.Sscanf(path, storageDir+"/%d-%d-%d.chunk", &item.ArtifactID, &item.Start, &item.End); err != nil { 382 - return fmt.Errorf("parse content range error: %v", err) 383 - } 384 - chunks = append(chunks, &item) 385 - return nil 386 - }); err != nil { 387 - return err 388 - } 389 - // group chunks by artifact id 390 - chunksMap := make(map[int64][]*chunkItem) 391 - for _, c := range chunks { 392 - chunksMap[c.ArtifactID] = append(chunksMap[c.ArtifactID], c) 393 - } 394 - 395 - for artifactID, cs := range chunksMap { 396 - // get artifact to handle merged chunks 397 - artifact, err := actions.GetArtifactByID(ctx, cs[0].ArtifactID) 398 - if err != nil { 399 - return fmt.Errorf("get artifact error: %v", err) 400 - } 401 - 402 - sort.Slice(cs, func(i, j int) bool { 403 - return cs[i].Start < cs[j].Start 404 - }) 405 - 406 - allChunks := make([]*chunkItem, 0) 407 - startAt := int64(-1) 408 - // check if all chunks are uploaded and in order and clean repeated chunks 409 - for _, c := range cs { 410 - // startAt is -1 means this is the first chunk 411 - // previous c.ChunkEnd + 1 == c.ChunkStart means this chunk is in order 412 - // StartAt is not -1 and c.ChunkStart is not startAt + 1 means there is a chunk missing 413 - if c.Start == (startAt + 1) { 414 - allChunks = append(allChunks, c) 415 - startAt = c.End 416 - } 417 - } 418 - 419 - // if the last chunk.End + 1 is not equal to chunk.ChunkLength, means chunks are not uploaded completely 420 - if startAt+1 != artifact.FileCompressedSize { 421 - log.Debug("[artifact] chunks are not uploaded completely, artifact_id: %d", artifactID) 422 - break 423 - } 424 - 425 - // use multiReader 426 - readers := make([]io.Reader, 0, len(allChunks)) 427 - closeReaders := func() { 428 - for _, r := range readers { 429 - _ = r.(io.Closer).Close() // it guarantees to be io.Closer by the following loop's Open function 430 - } 431 - readers = nil 432 - } 433 - defer closeReaders() 434 - 435 - for _, c := range allChunks { 436 - var readCloser io.ReadCloser 437 - if readCloser, err = ar.fs.Open(c.Path); err != nil { 438 - return fmt.Errorf("open chunk error: %v, %s", err, c.Path) 439 - } 440 - readers = append(readers, readCloser) 441 - } 442 - mergedReader := io.MultiReader(readers...) 443 - 444 - // if chunk is gzip, decompress it 445 - if artifact.ContentEncoding == "gzip" { 446 - var err error 447 - mergedReader, err = gzip.NewReader(mergedReader) 448 - if err != nil { 449 - return fmt.Errorf("gzip reader error: %v", err) 450 - } 451 - } 452 - 453 - // save merged file 454 - storagePath := fmt.Sprintf("%d/%d/%d.chunk", runID%255, artifactID%255, time.Now().UnixNano()) 455 - written, err := ar.fs.Save(storagePath, mergedReader, -1) 456 - if err != nil { 457 - return fmt.Errorf("save merged file error: %v", err) 458 - } 459 - if written != artifact.FileSize { 460 - return fmt.Errorf("merged file size is not equal to chunk length") 461 - } 462 - 463 - // save storage path to artifact 464 - log.Debug("[artifact] merge chunks to artifact: %d, %s", artifact.ID, storagePath) 465 - artifact.StoragePath = storagePath 466 - artifact.Status = actions.ArtifactStatusUploadConfirmed 467 - if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil { 468 - return fmt.Errorf("update artifact error: %v", err) 469 - } 470 - 471 - closeReaders() // close before delete 472 - 473 - // drop chunks 474 - for _, c := range cs { 475 - if err := ar.fs.Delete(c.Path); err != nil { 476 - return fmt.Errorf("delete chunk file error: %v", err) 477 - } 478 - } 479 - } 480 - return nil 481 - } 482 - 483 280 type ( 484 281 listArtifactsResponse struct { 485 282 Count int64 `json:"count"` ··· 492 289 ) 493 290 494 291 func (ar artifactRoutes) listArtifacts(ctx *ArtifactContext) { 495 - _, runID, ok := ar.validateRunID(ctx) 292 + _, runID, ok := validateRunID(ctx) 496 293 if !ok { 497 294 return 498 295 } ··· 503 300 ctx.Error(http.StatusInternalServerError, err.Error()) 504 301 return 505 302 } 303 + if len(artifacts) == 0 { 304 + log.Debug("[artifact] handleListArtifacts, no artifacts") 305 + ctx.Error(http.StatusNotFound) 306 + return 307 + } 506 308 507 - artficatsData := make([]listArtifactsResponseItem, 0, len(artifacts)) 508 - for _, a := range artifacts { 509 - artficatsData = append(artficatsData, listArtifactsResponseItem{ 510 - Name: a.ArtifactName, 511 - FileContainerResourceURL: ar.buildArtifactURL(runID, a.ID, "path"), 512 - }) 309 + var ( 310 + items []listArtifactsResponseItem 311 + values = make(map[string]bool) 312 + ) 313 + 314 + for _, art := range artifacts { 315 + if values[art.ArtifactName] { 316 + continue 317 + } 318 + artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(art.ArtifactName))) 319 + item := listArtifactsResponseItem{ 320 + Name: art.ArtifactName, 321 + FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "download_url"), 322 + } 323 + items = append(items, item) 324 + values[art.ArtifactName] = true 325 + 326 + log.Debug("[artifact] handleListArtifacts, name: %s, url: %s", item.Name, item.FileContainerResourceURL) 513 327 } 328 + 514 329 respData := listArtifactsResponse{ 515 - Count: int64(len(artficatsData)), 516 - Value: artficatsData, 330 + Count: int64(len(items)), 331 + Value: items, 517 332 } 518 333 ctx.JSON(http.StatusOK, respData) 519 334 } ··· 529 344 } 530 345 ) 531 346 347 + // getDownloadArtifactURL generates download url for each artifact 532 348 func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) { 533 - _, runID, ok := ar.validateRunID(ctx) 349 + _, runID, ok := validateRunID(ctx) 534 350 if !ok { 535 351 return 536 352 } 537 353 538 - artifactID := ctx.ParamsInt64("artifact_id") 539 - artifact, err := actions.GetArtifactByID(ctx, artifactID) 540 - if errors.Is(err, util.ErrNotExist) { 541 - log.Error("Error getting artifact: %v", err) 542 - ctx.Error(http.StatusNotFound, err.Error()) 354 + itemPath := util.PathJoinRel(ctx.Req.URL.Query().Get("itemPath")) 355 + if !validateArtifactHash(ctx, itemPath) { 543 356 return 544 - } else if err != nil { 545 - log.Error("Error getting artifact: %v", err) 357 + } 358 + 359 + artifacts, err := actions.ListArtifactsByRunIDAndArtifactName(ctx, runID, itemPath) 360 + if err != nil { 361 + log.Error("Error getting artifacts: %v", err) 546 362 ctx.Error(http.StatusInternalServerError, err.Error()) 547 363 return 548 364 } 549 - downloadURL := ar.buildArtifactURL(runID, artifact.ID, "download") 550 - itemPath := util.PathJoinRel(ctx.Req.URL.Query().Get("itemPath")) 551 - respData := downloadArtifactResponse{ 552 - Value: []downloadArtifactResponseItem{{ 365 + if len(artifacts) == 0 { 366 + log.Debug("[artifact] getDownloadArtifactURL, no artifacts") 367 + ctx.Error(http.StatusNotFound) 368 + return 369 + } 370 + 371 + if itemPath != artifacts[0].ArtifactName { 372 + log.Error("Error dismatch artifact name, itemPath: %v, artifact: %v", itemPath, artifacts[0].ArtifactName) 373 + ctx.Error(http.StatusBadRequest, "Error dismatch artifact name") 374 + return 375 + } 376 + 377 + var items []downloadArtifactResponseItem 378 + for _, artifact := range artifacts { 379 + downloadURL := ar.buildArtifactURL(runID, strconv.FormatInt(artifact.ID, 10), "download") 380 + item := downloadArtifactResponseItem{ 553 381 Path: util.PathJoinRel(itemPath, artifact.ArtifactPath), 554 382 ItemType: "file", 555 383 ContentLocation: downloadURL, 556 - }}, 384 + } 385 + log.Debug("[artifact] getDownloadArtifactURL, path: %s, url: %s", item.Path, item.ContentLocation) 386 + items = append(items, item) 387 + } 388 + respData := downloadArtifactResponse{ 389 + Value: items, 557 390 } 558 391 ctx.JSON(http.StatusOK, respData) 559 392 } 560 393 394 + // downloadArtifact downloads artifact content 561 395 func (ar artifactRoutes) downloadArtifact(ctx *ArtifactContext) { 562 - _, runID, ok := ar.validateRunID(ctx) 396 + _, runID, ok := validateRunID(ctx) 563 397 if !ok { 564 398 return 565 399 } ··· 589 423 } 590 424 defer fd.Close() 591 425 592 - if strings.HasSuffix(artifact.ArtifactPath, ".gz") { 426 + // if artifact is compressed, set content-encoding header to gzip 427 + if artifact.ContentEncoding == "gzip" { 593 428 ctx.Resp.Header().Set("Content-Encoding", "gzip") 594 429 } 430 + log.Debug("[artifact] downloadArtifact, name: %s, path: %s, storage: %s, size: %d", artifact.ArtifactName, artifact.ArtifactPath, artifact.StoragePath, artifact.FileSize) 595 431 ctx.ServeContent(fd, &context.ServeHeaderOptions{ 596 432 Filename: artifact.ArtifactName, 597 433 LastModified: artifact.CreatedUnix.AsLocalTime(),

+187

routers/api/actions/artifacts_chunks.go

··· 1 + // Copyright 2023 The Gitea Authors. All rights reserved. 2 + // SPDX-License-Identifier: MIT 3 + 4 + package actions 5 + 6 + import ( 7 + "crypto/md5" 8 + "encoding/base64" 9 + "fmt" 10 + "io" 11 + "sort" 12 + "time" 13 + 14 + "code.gitea.io/gitea/models/actions" 15 + "code.gitea.io/gitea/modules/log" 16 + "code.gitea.io/gitea/modules/storage" 17 + ) 18 + 19 + func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext, 20 + artifact *actions.ActionArtifact, 21 + contentSize, runID int64, 22 + ) (int64, error) { 23 + // parse content-range header, format: bytes 0-1023/146515 24 + contentRange := ctx.Req.Header.Get("Content-Range") 25 + start, end, length := int64(0), int64(0), int64(0) 26 + if _, err := fmt.Sscanf(contentRange, "bytes %d-%d/%d", &start, &end, &length); err != nil { 27 + return -1, fmt.Errorf("parse content range error: %v", err) 28 + } 29 + // build chunk store path 30 + storagePath := fmt.Sprintf("tmp%d/%d-%d-%d.chunk", runID, artifact.ID, start, end) 31 + // use io.TeeReader to avoid reading all body to md5 sum. 32 + // it writes data to hasher after reading end 33 + // if hash is not matched, delete the read-end result 34 + hasher := md5.New() 35 + r := io.TeeReader(ctx.Req.Body, hasher) 36 + // save chunk to storage 37 + writtenSize, err := st.Save(storagePath, r, -1) 38 + if err != nil { 39 + return -1, fmt.Errorf("save chunk to storage error: %v", err) 40 + } 41 + // check md5 42 + reqMd5String := ctx.Req.Header.Get(artifactXActionsResultsMD5Header) 43 + chunkMd5String := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) 44 + log.Info("[artifact] check chunk md5, sum: %s, header: %s", chunkMd5String, reqMd5String) 45 + // if md5 not match, delete the chunk 46 + if reqMd5String != chunkMd5String || writtenSize != contentSize { 47 + if err := st.Delete(storagePath); err != nil { 48 + log.Error("Error deleting chunk: %s, %v", storagePath, err) 49 + } 50 + return -1, fmt.Errorf("md5 not match") 51 + } 52 + log.Info("[artifact] save chunk %s, size: %d, artifact id: %d, start: %d, end: %d", 53 + storagePath, contentSize, artifact.ID, start, end) 54 + // return chunk total size 55 + return length, nil 56 + } 57 + 58 + type chunkFileItem struct { 59 + ArtifactID int64 60 + Start int64 61 + End int64 62 + Path string 63 + } 64 + 65 + func listChunksByRunID(st storage.ObjectStorage, runID int64) (map[int64][]*chunkFileItem, error) { 66 + storageDir := fmt.Sprintf("tmp%d", runID) 67 + var chunks []*chunkFileItem 68 + if err := st.IterateObjects(storageDir, func(path string, obj storage.Object) error { 69 + item := chunkFileItem{Path: path} 70 + if _, err := fmt.Sscanf(path, storageDir+"/%d-%d-%d.chunk", &item.ArtifactID, &item.Start, &item.End); err != nil { 71 + return fmt.Errorf("parse content range error: %v", err) 72 + } 73 + chunks = append(chunks, &item) 74 + return nil 75 + }); err != nil { 76 + return nil, err 77 + } 78 + // chunks group by artifact id 79 + chunksMap := make(map[int64][]*chunkFileItem) 80 + for _, c := range chunks { 81 + chunksMap[c.ArtifactID] = append(chunksMap[c.ArtifactID], c) 82 + } 83 + return chunksMap, nil 84 + } 85 + 86 + func mergeChunksForRun(ctx *ArtifactContext, st storage.ObjectStorage, runID int64, artifactName string) error { 87 + // read all db artifacts by name 88 + artifacts, err := actions.ListArtifactsByRunIDAndName(ctx, runID, artifactName) 89 + if err != nil { 90 + return err 91 + } 92 + // read all uploading chunks from storage 93 + chunksMap, err := listChunksByRunID(st, runID) 94 + if err != nil { 95 + return err 96 + } 97 + // range db artifacts to merge chunks 98 + for _, art := range artifacts { 99 + chunks, ok := chunksMap[art.ID] 100 + if !ok { 101 + log.Debug("artifact %d chunks not found", art.ID) 102 + continue 103 + } 104 + if err := mergeChunksForArtifact(ctx, chunks, st, art); err != nil { 105 + return err 106 + } 107 + } 108 + return nil 109 + } 110 + 111 + func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st storage.ObjectStorage, artifact *actions.ActionArtifact) error { 112 + sort.Slice(chunks, func(i, j int) bool { 113 + return chunks[i].Start < chunks[j].Start 114 + }) 115 + allChunks := make([]*chunkFileItem, 0) 116 + startAt := int64(-1) 117 + // check if all chunks are uploaded and in order and clean repeated chunks 118 + for _, c := range chunks { 119 + // startAt is -1 means this is the first chunk 120 + // previous c.ChunkEnd + 1 == c.ChunkStart means this chunk is in order 121 + // StartAt is not -1 and c.ChunkStart is not startAt + 1 means there is a chunk missing 122 + if c.Start == (startAt + 1) { 123 + allChunks = append(allChunks, c) 124 + startAt = c.End 125 + } 126 + } 127 + // if the last chunk.End + 1 is not equal to chunk.ChunkLength, means chunks are not uploaded completely 128 + if startAt+1 != artifact.FileCompressedSize { 129 + log.Debug("[artifact] chunks are not uploaded completely, artifact_id: %d", artifact.ID) 130 + return nil 131 + } 132 + // use multiReader 133 + readers := make([]io.Reader, 0, len(allChunks)) 134 + closeReaders := func() { 135 + for _, r := range readers { 136 + _ = r.(io.Closer).Close() // it guarantees to be io.Closer by the following loop's Open function 137 + } 138 + readers = nil 139 + } 140 + defer closeReaders() 141 + for _, c := range allChunks { 142 + var readCloser io.ReadCloser 143 + var err error 144 + if readCloser, err = st.Open(c.Path); err != nil { 145 + return fmt.Errorf("open chunk error: %v, %s", err, c.Path) 146 + } 147 + readers = append(readers, readCloser) 148 + } 149 + mergedReader := io.MultiReader(readers...) 150 + 151 + // if chunk is gzip, use gz as extension 152 + // download-artifact action will use content-encoding header to decide if it should decompress the file 153 + extension := "chunk" 154 + if artifact.ContentEncoding == "gzip" { 155 + extension = "chunk.gz" 156 + } 157 + 158 + // save merged file 159 + storagePath := fmt.Sprintf("%d/%d/%d.%s", artifact.RunID%255, artifact.ID%255, time.Now().UnixNano(), extension) 160 + written, err := st.Save(storagePath, mergedReader, -1) 161 + if err != nil { 162 + return fmt.Errorf("save merged file error: %v", err) 163 + } 164 + if written != artifact.FileCompressedSize { 165 + return fmt.Errorf("merged file size is not equal to chunk length") 166 + } 167 + 168 + defer func() { 169 + closeReaders() // close before delete 170 + // drop chunks 171 + for _, c := range chunks { 172 + if err := st.Delete(c.Path); err != nil { 173 + log.Warn("Error deleting chunk: %s, %v", c.Path, err) 174 + } 175 + } 176 + }() 177 + 178 + // save storage path to artifact 179 + log.Debug("[artifact] merge chunks to artifact: %d, %s", artifact.ID, storagePath) 180 + artifact.StoragePath = storagePath 181 + artifact.Status = actions.ArtifactStatusUploadConfirmed 182 + if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil { 183 + return fmt.Errorf("update artifact error: %v", err) 184 + } 185 + 186 + return nil 187 + }

+82

routers/api/actions/artifacts_utils.go

··· 1 + // Copyright 2023 The Gitea Authors. All rights reserved. 2 + // SPDX-License-Identifier: MIT 3 + 4 + package actions 5 + 6 + import ( 7 + "crypto/md5" 8 + "fmt" 9 + "net/http" 10 + "strconv" 11 + "strings" 12 + 13 + "code.gitea.io/gitea/models/actions" 14 + "code.gitea.io/gitea/modules/log" 15 + "code.gitea.io/gitea/modules/util" 16 + ) 17 + 18 + const ( 19 + artifactXTfsFileLengthHeader = "x-tfs-filelength" 20 + artifactXActionsResultsMD5Header = "x-actions-results-md5" 21 + ) 22 + 23 + // The rules are from https://github.com/actions/toolkit/blob/main/packages/artifact/src/internal/path-and-artifact-name-validation.ts#L32 24 + var invalidArtifactNameChars = strings.Join([]string{"\\", "/", "\"", ":", "<", ">", "|", "*", "?", "\r", "\n"}, "") 25 + 26 + func validateArtifactName(ctx *ArtifactContext, artifactName string) bool { 27 + if strings.ContainsAny(artifactName, invalidArtifactNameChars) { 28 + log.Error("Error checking artifact name contains invalid character") 29 + ctx.Error(http.StatusBadRequest, "Error checking artifact name contains invalid character") 30 + return false 31 + } 32 + return true 33 + } 34 + 35 + func validateRunID(ctx *ArtifactContext) (*actions.ActionTask, int64, bool) { 36 + task := ctx.ActionTask 37 + runID := ctx.ParamsInt64("run_id") 38 + if task.Job.RunID != runID { 39 + log.Error("Error runID not match") 40 + ctx.Error(http.StatusBadRequest, "run-id does not match") 41 + return nil, 0, false 42 + } 43 + return task, runID, true 44 + } 45 + 46 + func validateArtifactHash(ctx *ArtifactContext, artifactName string) bool { 47 + paramHash := ctx.Params("artifact_hash") 48 + // use artifact name to create upload url 49 + artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(artifactName))) 50 + if paramHash == artifactHash { 51 + return true 52 + } 53 + log.Error("Invalid artifact hash: %s", paramHash) 54 + ctx.Error(http.StatusBadRequest, "Invalid artifact hash") 55 + return false 56 + } 57 + 58 + func parseArtifactItemPath(ctx *ArtifactContext) (string, string, bool) { 59 + // itemPath is generated from upload-artifact action 60 + // it's formatted as {artifact_name}/{artfict_path_in_runner} 61 + itemPath := util.PathJoinRel(ctx.Req.URL.Query().Get("itemPath")) 62 + artifactName := strings.Split(itemPath, "/")[0] 63 + artifactPath := strings.TrimPrefix(itemPath, artifactName+"/") 64 + if !validateArtifactHash(ctx, artifactName) { 65 + return "", "", false 66 + } 67 + if !validateArtifactName(ctx, artifactName) { 68 + return "", "", false 69 + } 70 + return artifactName, artifactPath, true 71 + } 72 + 73 + // getUploadFileSize returns the size of the file to be uploaded. 74 + // The raw size is the size of the file as reported by the header X-TFS-FileLength. 75 + func getUploadFileSize(ctx *ArtifactContext) (int64, int64, error) { 76 + contentLength := ctx.Req.ContentLength 77 + xTfsLength, _ := strconv.ParseInt(ctx.Req.Header.Get(artifactXTfsFileLengthHeader), 10, 64) 78 + if xTfsLength > 0 { 79 + return xTfsLength, contentLength, nil 80 + } 81 + return contentLength, contentLength, nil 82 + }

+45 -21

routers/web/repo/actions/view.go

··· 4 4 package actions 5 5 6 6 import ( 7 + "archive/zip" 8 + "compress/gzip" 7 9 "context" 8 10 "errors" 9 11 "fmt" 12 + "io" 10 13 "net/http" 14 + "net/url" 11 15 "strings" 12 16 "time" 13 17 ··· 479 483 type ArtifactsViewItem struct { 480 484 Name string `json:"name"` 481 485 Size int64 `json:"size"` 482 - ID int64 `json:"id"` 483 486 } 484 487 485 488 func ArtifactsView(ctx *context_module.Context) { ··· 493 496 ctx.Error(http.StatusInternalServerError, err.Error()) 494 497 return 495 498 } 496 - artifacts, err := actions_model.ListUploadedArtifactsByRunID(ctx, run.ID) 499 + artifacts, err := actions_model.ListUploadedArtifactsMeta(ctx, run.ID) 497 500 if err != nil { 498 501 ctx.Error(http.StatusInternalServerError, err.Error()) 499 502 return ··· 505 508 artifactsResponse.Artifacts = append(artifactsResponse.Artifacts, &ArtifactsViewItem{ 506 509 Name: art.ArtifactName, 507 510 Size: art.FileSize, 508 - ID: art.ID, 509 511 }) 510 512 } 511 513 ctx.JSON(http.StatusOK, artifactsResponse) ··· 513 515 514 516 func ArtifactsDownloadView(ctx *context_module.Context) { 515 517 runIndex := ctx.ParamsInt64("run") 516 - artifactID := ctx.ParamsInt64("id") 518 + artifactName := ctx.Params("artifact_name") 517 519 518 - artifact, err := actions_model.GetArtifactByID(ctx, artifactID) 519 - if errors.Is(err, util.ErrNotExist) { 520 - ctx.Error(http.StatusNotFound, err.Error()) 521 - } else if err != nil { 522 - ctx.Error(http.StatusInternalServerError, err.Error()) 523 - return 524 - } 525 520 run, err := actions_model.GetRunByIndex(ctx, ctx.Repo.Repository.ID, runIndex) 526 521 if err != nil { 527 522 if errors.Is(err, util.ErrNotExist) { ··· 531 526 ctx.Error(http.StatusInternalServerError, err.Error()) 532 527 return 533 528 } 534 - if artifact.RunID != run.ID { 535 - ctx.Error(http.StatusNotFound, "artifact not found") 536 - return 537 - } 538 529 539 - f, err := storage.ActionsArtifacts.Open(artifact.StoragePath) 530 + artifacts, err := actions_model.ListArtifactsByRunIDAndName(ctx, run.ID, artifactName) 540 531 if err != nil { 541 532 ctx.Error(http.StatusInternalServerError, err.Error()) 542 533 return 543 534 } 544 - defer f.Close() 535 + if len(artifacts) == 0 { 536 + ctx.Error(http.StatusNotFound, "artifact not found") 537 + return 538 + } 539 + 540 + ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s.zip; filename*=UTF-8''%s.zip", url.PathEscape(artifactName), artifactName)) 545 541 546 - ctx.ServeContent(f, &context_module.ServeHeaderOptions{ 547 - Filename: artifact.ArtifactName, 548 - LastModified: artifact.CreatedUnix.AsLocalTime(), 549 - }) 542 + writer := zip.NewWriter(ctx.Resp) 543 + defer writer.Close() 544 + for _, art := range artifacts { 545 + 546 + f, err := storage.ActionsArtifacts.Open(art.StoragePath) 547 + if err != nil { 548 + ctx.Error(http.StatusInternalServerError, err.Error()) 549 + return 550 + } 551 + 552 + var r io.ReadCloser 553 + if art.ContentEncoding == "gzip" { 554 + r, err = gzip.NewReader(f) 555 + if err != nil { 556 + ctx.Error(http.StatusInternalServerError, err.Error()) 557 + return 558 + } 559 + } else { 560 + r = f 561 + } 562 + defer r.Close() 563 + 564 + w, err := writer.Create(art.ArtifactPath) 565 + if err != nil { 566 + ctx.Error(http.StatusInternalServerError, err.Error()) 567 + return 568 + } 569 + if _, err := io.Copy(w, r); err != nil { 570 + ctx.Error(http.StatusInternalServerError, err.Error()) 571 + return 572 + } 573 + } 550 574 }

+1 -1

routers/web/web.go

··· 1210 1210 m.Post("/cancel", reqRepoActionsWriter, actions.Cancel) 1211 1211 m.Post("/approve", reqRepoActionsWriter, actions.Approve) 1212 1212 m.Post("/artifacts", actions.ArtifactsView) 1213 - m.Get("/artifacts/{id}", actions.ArtifactsDownloadView) 1213 + m.Get("/artifacts/{artifact_name}", actions.ArtifactsDownloadView) 1214 1214 m.Post("/rerun", reqRepoActionsWriter, actions.RerunAll) 1215 1215 }) 1216 1216 }, reqRepoActionsReader, actions.MustEnableActions)

+150 -39

tests/integration/api_actions_artifact_test.go

··· 13 13 "github.com/stretchr/testify/assert" 14 14 ) 15 15 16 - func TestActionsArtifactUpload(t *testing.T) { 17 - defer tests.PrepareTestEnv(t)() 16 + type uploadArtifactResponse struct { 17 + FileContainerResourceURL string `json:"fileContainerResourceUrl"` 18 + } 18 19 19 - type uploadArtifactResponse struct { 20 - FileContainerResourceURL string `json:"fileContainerResourceUrl"` 21 - } 20 + type getUploadArtifactRequest struct { 21 + Type string 22 + Name string 23 + } 22 24 23 - type getUploadArtifactRequest struct { 24 - Type string 25 - Name string 26 - } 25 + func TestActionsArtifactUploadSingleFile(t *testing.T) { 26 + defer tests.PrepareTestEnv(t)() 27 27 28 28 // acquire artifact upload url 29 29 req := NewRequestWithJSON(t, "POST", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts", getUploadArtifactRequest{ ··· 52 52 t.Logf("Create artifact confirm") 53 53 54 54 // confirm artifact upload 55 - req = NewRequest(t, "PATCH", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts") 55 + req = NewRequest(t, "PATCH", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts?artifactName=artifact") 56 56 req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 57 57 MakeRequest(t, req, http.StatusOK) 58 58 } 59 59 60 - func TestActionsArtifactUploadNotExist(t *testing.T) { 60 + func TestActionsArtifactUploadInvalidHash(t *testing.T) { 61 61 defer tests.PrepareTestEnv(t)() 62 62 63 63 // artifact id 54321 not exist 64 - url := "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts/54321/upload?itemPath=artifact/abc.txt" 64 + url := "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts/8e5b948a454515dbabfc7eb718ddddddd/upload?itemPath=artifact/abc.txt" 65 65 body := strings.Repeat("A", 1024) 66 66 req := NewRequestWithBody(t, "PUT", url, strings.NewReader(body)) 67 67 req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 68 68 req.Header.Add("Content-Range", "bytes 0-1023/1024") 69 69 req.Header.Add("x-tfs-filelength", "1024") 70 70 req.Header.Add("x-actions-results-md5", "1HsSe8LeLWh93ILaw1TEFQ==") // base64(md5(body)) 71 - MakeRequest(t, req, http.StatusNotFound) 71 + resp := MakeRequest(t, req, http.StatusBadRequest) 72 + assert.Contains(t, resp.Body.String(), "Invalid artifact hash") 72 73 } 73 74 74 - func TestActionsArtifactConfirmUpload(t *testing.T) { 75 + func TestActionsArtifactConfirmUploadWithoutName(t *testing.T) { 75 76 defer tests.PrepareTestEnv(t)() 76 77 77 78 req := NewRequest(t, "PATCH", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts") 78 79 req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 79 - resp := MakeRequest(t, req, http.StatusOK) 80 - assert.Contains(t, resp.Body.String(), "success") 80 + resp := MakeRequest(t, req, http.StatusBadRequest) 81 + assert.Contains(t, resp.Body.String(), "artifact name is empty") 81 82 } 82 83 83 84 func TestActionsArtifactUploadWithoutToken(t *testing.T) { ··· 87 88 MakeRequest(t, req, http.StatusUnauthorized) 88 89 } 89 90 91 + type ( 92 + listArtifactsResponseItem struct { 93 + Name string `json:"name"` 94 + FileContainerResourceURL string `json:"fileContainerResourceUrl"` 95 + } 96 + listArtifactsResponse struct { 97 + Count int64 `json:"count"` 98 + Value []listArtifactsResponseItem `json:"value"` 99 + } 100 + downloadArtifactResponseItem struct { 101 + Path string `json:"path"` 102 + ItemType string `json:"itemType"` 103 + ContentLocation string `json:"contentLocation"` 104 + } 105 + downloadArtifactResponse struct { 106 + Value []downloadArtifactResponseItem `json:"value"` 107 + } 108 + ) 109 + 90 110 func TestActionsArtifactDownload(t *testing.T) { 91 111 defer tests.PrepareTestEnv(t)() 92 112 93 - type ( 94 - listArtifactsResponseItem struct { 95 - Name string `json:"name"` 96 - FileContainerResourceURL string `json:"fileContainerResourceUrl"` 97 - } 98 - listArtifactsResponse struct { 99 - Count int64 `json:"count"` 100 - Value []listArtifactsResponseItem `json:"value"` 101 - } 102 - ) 103 - 104 113 req := NewRequest(t, "GET", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts") 105 114 req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 106 115 resp := MakeRequest(t, req, http.StatusOK) ··· 110 119 assert.Equal(t, "artifact", listResp.Value[0].Name) 111 120 assert.Contains(t, listResp.Value[0].FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts") 112 121 113 - type ( 114 - downloadArtifactResponseItem struct { 115 - Path string `json:"path"` 116 - ItemType string `json:"itemType"` 117 - ContentLocation string `json:"contentLocation"` 118 - } 119 - downloadArtifactResponse struct { 120 - Value []downloadArtifactResponseItem `json:"value"` 121 - } 122 - ) 123 - 124 122 idx := strings.Index(listResp.Value[0].FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/") 125 - url := listResp.Value[0].FileContainerResourceURL[idx+1:] 123 + url := listResp.Value[0].FileContainerResourceURL[idx+1:] + "?itemPath=artifact" 126 124 req = NewRequest(t, "GET", url) 127 125 req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 128 126 resp = MakeRequest(t, req, http.StatusOK) ··· 141 139 body := strings.Repeat("A", 1024) 142 140 assert.Equal(t, resp.Body.String(), body) 143 141 } 142 + 143 + func TestActionsArtifactUploadMultipleFile(t *testing.T) { 144 + defer tests.PrepareTestEnv(t)() 145 + 146 + const testArtifactName = "multi-files" 147 + 148 + // acquire artifact upload url 149 + req := NewRequestWithJSON(t, "POST", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts", getUploadArtifactRequest{ 150 + Type: "actions_storage", 151 + Name: testArtifactName, 152 + }) 153 + req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 154 + resp := MakeRequest(t, req, http.StatusOK) 155 + var uploadResp uploadArtifactResponse 156 + DecodeJSON(t, resp, &uploadResp) 157 + assert.Contains(t, uploadResp.FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts") 158 + 159 + type uploadingFile struct { 160 + Path string 161 + Content string 162 + MD5 string 163 + } 164 + 165 + files := []uploadingFile{ 166 + { 167 + Path: "abc.txt", 168 + Content: strings.Repeat("A", 1024), 169 + MD5: "1HsSe8LeLWh93ILaw1TEFQ==", 170 + }, 171 + { 172 + Path: "xyz/def.txt", 173 + Content: strings.Repeat("B", 1024), 174 + MD5: "6fgADK/7zjadf+6cB9Q1CQ==", 175 + }, 176 + } 177 + 178 + for _, f := range files { 179 + // get upload url 180 + idx := strings.Index(uploadResp.FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/") 181 + url := uploadResp.FileContainerResourceURL[idx:] + "?itemPath=" + testArtifactName + "/" + f.Path 182 + 183 + // upload artifact chunk 184 + req = NewRequestWithBody(t, "PUT", url, strings.NewReader(f.Content)) 185 + req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 186 + req.Header.Add("Content-Range", "bytes 0-1023/1024") 187 + req.Header.Add("x-tfs-filelength", "1024") 188 + req.Header.Add("x-actions-results-md5", f.MD5) // base64(md5(body)) 189 + MakeRequest(t, req, http.StatusOK) 190 + } 191 + 192 + t.Logf("Create artifact confirm") 193 + 194 + // confirm artifact upload 195 + req = NewRequest(t, "PATCH", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts?artifactName="+testArtifactName) 196 + req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 197 + MakeRequest(t, req, http.StatusOK) 198 + } 199 + 200 + func TestActionsArtifactDownloadMultiFiles(t *testing.T) { 201 + defer tests.PrepareTestEnv(t)() 202 + 203 + const testArtifactName = "multi-files" 204 + 205 + req := NewRequest(t, "GET", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts") 206 + req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 207 + resp := MakeRequest(t, req, http.StatusOK) 208 + var listResp listArtifactsResponse 209 + DecodeJSON(t, resp, &listResp) 210 + assert.Equal(t, int64(2), listResp.Count) 211 + 212 + var fileContainerResourceURL string 213 + for _, v := range listResp.Value { 214 + if v.Name == testArtifactName { 215 + fileContainerResourceURL = v.FileContainerResourceURL 216 + break 217 + } 218 + } 219 + assert.Contains(t, fileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts") 220 + 221 + idx := strings.Index(fileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/") 222 + url := fileContainerResourceURL[idx+1:] + "?itemPath=" + testArtifactName 223 + req = NewRequest(t, "GET", url) 224 + req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 225 + resp = MakeRequest(t, req, http.StatusOK) 226 + var downloadResp downloadArtifactResponse 227 + DecodeJSON(t, resp, &downloadResp) 228 + assert.Len(t, downloadResp.Value, 2) 229 + 230 + downloads := [][]string{{"multi-files/abc.txt", "A"}, {"multi-files/xyz/def.txt", "B"}} 231 + for _, v := range downloadResp.Value { 232 + var bodyChar string 233 + var path string 234 + for _, d := range downloads { 235 + if v.Path == d[0] { 236 + path = d[0] 237 + bodyChar = d[1] 238 + break 239 + } 240 + } 241 + value := v 242 + assert.Equal(t, path, value.Path) 243 + assert.Equal(t, "file", value.ItemType) 244 + assert.Contains(t, value.ContentLocation, "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts") 245 + 246 + idx = strings.Index(value.ContentLocation, "/api/actions_pipeline/_apis/pipelines/") 247 + url = value.ContentLocation[idx:] 248 + req = NewRequest(t, "GET", url) 249 + req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") 250 + resp = MakeRequest(t, req, http.StatusOK) 251 + body := strings.Repeat(bodyChar, 1024) 252 + assert.Equal(t, resp.Body.String(), body) 253 + } 254 + }

+2 -2

web_src/js/components/RepoActionView.vue

··· 49 49 {{ locale.artifactsTitle }} 50 50 </div> 51 51 <ul class="job-artifacts-list"> 52 - <li class="job-artifacts-item" v-for="artifact in artifacts" :key="artifact.id"> 53 - <a class="job-artifacts-link" target="_blank" :href="run.link+'/artifacts/'+artifact.id"> 52 + <li class="job-artifacts-item" v-for="artifact in artifacts" :key="artifact.name"> 53 + <a class="job-artifacts-link" target="_blank" :href="run.link+'/artifacts/'+artifact.name"> 54 54 <SvgIcon name="octicon-file" class="ui text black job-artifacts-icon"/>{{ artifact.name }} 55 55 </a> 56 56 </li>

Configure Feed

Configure Feed