feat: support grouping by any path for arch package (#4903)

+34 -8

modules/packages/arch/metadata.go

··· 41 41 reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) 42 42 reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`) 43 43 rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`) 44 + 45 + magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD} 46 + magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A} 44 47 ) 45 48 46 49 type Package struct { 47 50 Name string `json:"name"` 48 51 Version string `json:"version"` // Includes version, release and epoch 52 + CompressType string `json:"compress_type"` 49 53 VersionMetadata VersionMetadata 50 54 FileMetadata FileMetadata 51 55 } ··· 89 93 if err != nil { 90 94 return nil, err 91 95 } 92 - zstd := archiver.NewTarZstd() 93 - err = zstd.Open(r, 0) 96 + header := make([]byte, 5) 97 + _, err = r.Read(header) 94 98 if err != nil { 95 99 return nil, err 96 100 } 97 - defer zstd.Close() 101 + _, err = r.Seek(0, io.SeekStart) 102 + if err != nil { 103 + return nil, err 104 + } 105 + 106 + var tarball archiver.Reader 107 + var tarballType string 108 + if bytes.Equal(header[:len(magicZSTD)], magicZSTD) { 109 + tarballType = "zst" 110 + tarball = archiver.NewTarZstd() 111 + } else if bytes.Equal(header[:len(magicXZ)], magicXZ) { 112 + tarballType = "xz" 113 + tarball = archiver.NewTarXz() 114 + } else { 115 + return nil, errors.New("not supported compression") 116 + } 117 + err = tarball.Open(r, 0) 118 + if err != nil { 119 + return nil, err 120 + } 121 + defer tarball.Close() 98 122 99 123 var pkg *Package 100 124 var mtree bool 101 125 102 126 for { 103 - f, err := zstd.Read() 127 + f, err := tarball.Read() 104 128 if err == io.EOF { 105 129 break 106 130 } ··· 111 135 112 136 switch f.Name() { 113 137 case ".PKGINFO": 114 - pkg, err = ParsePackageInfo(f) 138 + pkg, err = ParsePackageInfo(tarballType, f) 115 139 if err != nil { 116 140 return nil, err 117 141 } ··· 137 161 138 162 // ParsePackageInfo Function that accepts reader for .PKGINFO file from package archive, 139 163 // validates all field according to PKGBUILD spec and returns package. 140 - func ParsePackageInfo(r io.Reader) (*Package, error) { 141 - p := &Package{} 164 + func ParsePackageInfo(compressType string, r io.Reader) (*Package, error) { 165 + p := &Package{ 166 + CompressType: compressType, 167 + } 142 168 143 169 scanner := bufio.NewScanner(r) 144 170 for scanner.Scan() { ··· 281 307 // Desc Create pacman package description file. 282 308 func (p *Package) Desc() string { 283 309 entries := []string{ 284 - "FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch), 310 + "FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.%s", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType), 285 311 "NAME", p.Name, 286 312 "BASE", p.VersionMetadata.Base, 287 313 "VERSION", p.Version,

+7 -5

modules/packages/arch/metadata_test.go

··· 158 158 makedepend = cmake 159 159 backup = usr/bin/paket1 160 160 ` 161 - p, err := ParsePackageInfo(strings.NewReader(PKGINFO)) 161 + p, err := ParsePackageInfo("zst", strings.NewReader(PKGINFO)) 162 162 require.NoError(t, err) 163 163 require.Equal(t, Package{ 164 - Name: "a", 165 - Version: "1-2", 164 + CompressType: "zst", 165 + Name: "a", 166 + Version: "1-2", 166 167 VersionMetadata: VersionMetadata{ 167 168 Base: "b", 168 169 Description: "comment", ··· 417 418 ` 418 419 419 420 md := &Package{ 420 - Name: "zstd", 421 - Version: "1.5.5-1", 421 + CompressType: "zst", 422 + Name: "zstd", 423 + Version: "1.5.5-1", 422 424 VersionMetadata: VersionMetadata{ 423 425 Base: "zstd", 424 426 Description: "Zstandard - Fast real-time compression algorithm",

+53 -4

routers/api/packages/api.go

··· 143 143 r.Head("", arch.GetRepositoryKey) 144 144 r.Get("", arch.GetRepositoryKey) 145 145 }) 146 - r.Group("/{distro}", func() { 147 - r.Put("", reqPackageAccess(perm.AccessModeWrite), arch.PushPackage) 148 - r.Get("/{arch}/{file}", arch.GetPackageOrDB) 149 - r.Delete("/{package}/{version}", reqPackageAccess(perm.AccessModeWrite), arch.RemovePackage) 146 + 147 + r.Methods("HEAD,GET,PUT,DELETE", "*", func(ctx *context.Context) { 148 + pathGroups := strings.Split(strings.Trim(ctx.Params("*"), "/"), "/") 149 + groupLen := len(pathGroups) 150 + isGetHead := ctx.Req.Method == "HEAD" || ctx.Req.Method == "GET" 151 + isPut := ctx.Req.Method == "PUT" 152 + isDelete := ctx.Req.Method == "DELETE" 153 + if isGetHead { 154 + if groupLen < 2 { 155 + ctx.Status(http.StatusNotFound) 156 + return 157 + } 158 + if groupLen == 2 { 159 + ctx.SetParams("group", "") 160 + ctx.SetParams("arch", pathGroups[0]) 161 + ctx.SetParams("file", pathGroups[1]) 162 + } else { 163 + ctx.SetParams("group", strings.Join(pathGroups[:groupLen-2], "/")) 164 + ctx.SetParams("arch", pathGroups[groupLen-2]) 165 + ctx.SetParams("file", pathGroups[groupLen-1]) 166 + } 167 + arch.GetPackageOrDB(ctx) 168 + return 169 + } else if isPut { 170 + ctx.SetParams("group", strings.Join(pathGroups, "/")) 171 + reqPackageAccess(perm.AccessModeWrite)(ctx) 172 + if ctx.Written() { 173 + return 174 + } 175 + arch.PushPackage(ctx) 176 + return 177 + } else if isDelete { 178 + if groupLen < 2 { 179 + ctx.Status(http.StatusBadRequest) 180 + return 181 + } 182 + if groupLen == 2 { 183 + ctx.SetParams("group", "") 184 + ctx.SetParams("package", pathGroups[0]) 185 + ctx.SetParams("version", pathGroups[1]) 186 + } else { 187 + ctx.SetParams("group", strings.Join(pathGroups[:groupLen-2], "/")) 188 + ctx.SetParams("package", pathGroups[groupLen-2]) 189 + ctx.SetParams("version", pathGroups[groupLen-1]) 190 + } 191 + reqPackageAccess(perm.AccessModeWrite)(ctx) 192 + if ctx.Written() { 193 + return 194 + } 195 + arch.RemovePackage(ctx) 196 + return 197 + } 198 + ctx.Status(http.StatusNotFound) 150 199 }) 151 200 }, reqPackageAccess(perm.AccessModeRead)) 152 201 r.Group("/cargo", func() {

+26 -24

routers/api/packages/arch/arch.go

··· 9 9 "fmt" 10 10 "io" 11 11 "net/http" 12 + "regexp" 12 13 "strings" 13 14 14 15 packages_model "code.gitea.io/gitea/models/packages" ··· 19 20 "code.gitea.io/gitea/services/context" 20 21 packages_service "code.gitea.io/gitea/services/packages" 21 22 arch_service "code.gitea.io/gitea/services/packages/arch" 23 + ) 24 + 25 + var ( 26 + archPkgOrSig = regexp.MustCompile(`^.*\.pkg\.tar\.\w+(\.sig)*$`) 27 + archDBOrSig = regexp.MustCompile(`^.*.db(\.tar\.gz)*(\.sig)*$`) 22 28 ) 23 29 24 30 func apiError(ctx *context.Context, status int, obj any) { ··· 41 47 } 42 48 43 49 func PushPackage(ctx *context.Context) { 44 - distro := ctx.Params("distro") 50 + group := ctx.Params("group") 45 51 46 52 upload, needToClose, err := ctx.UploadStream() 47 53 if err != nil { ··· 61 67 62 68 p, err := arch_module.ParsePackage(buf) 63 69 if err != nil { 64 - apiError(ctx, http.StatusInternalServerError, err) 70 + apiError(ctx, http.StatusBadRequest, err) 65 71 return 66 72 } 67 73 ··· 97 103 properties := map[string]string{ 98 104 arch_module.PropertyDescription: p.Desc(), 99 105 arch_module.PropertyArch: p.FileMetadata.Arch, 100 - arch_module.PropertyDistribution: distro, 106 + arch_module.PropertyDistribution: group, 101 107 } 102 108 103 109 version, _, err := packages_service.CreatePackageOrAddFileToExisting( ··· 114 120 }, 115 121 &packages_service.PackageFileCreationInfo{ 116 122 PackageFileInfo: packages_service.PackageFileInfo{ 117 - Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch), 118 - CompositeKey: distro, 123 + Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.%s", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType), 124 + CompositeKey: group, 119 125 }, 120 126 OverwriteExisting: false, 121 127 IsLead: true, ··· 138 144 // add sign file 139 145 _, err = packages_service.AddFileToPackageVersionInternal(ctx, version, &packages_service.PackageFileCreationInfo{ 140 146 PackageFileInfo: packages_service.PackageFileInfo{ 141 - CompositeKey: distro, 142 - Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.zst.sig", p.Name, p.Version, p.FileMetadata.Arch), 147 + CompositeKey: group, 148 + Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.%s.sig", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType), 143 149 }, 144 150 OverwriteExisting: true, 145 151 IsLead: false, ··· 149 155 if err != nil { 150 156 apiError(ctx, http.StatusInternalServerError, err) 151 157 } 152 - if err = arch_service.BuildPacmanDB(ctx, ctx.Package.Owner.ID, distro, p.FileMetadata.Arch); err != nil { 158 + if err = arch_service.BuildPacmanDB(ctx, ctx.Package.Owner.ID, group, p.FileMetadata.Arch); err != nil { 153 159 apiError(ctx, http.StatusInternalServerError, err) 154 160 return 155 161 } ··· 158 164 159 165 func GetPackageOrDB(ctx *context.Context) { 160 166 var ( 161 - file = ctx.Params("file") 162 - distro = ctx.Params("distro") 163 - arch = ctx.Params("arch") 167 + file = ctx.Params("file") 168 + group = ctx.Params("group") 169 + arch = ctx.Params("arch") 164 170 ) 165 - 166 - if strings.HasSuffix(file, ".pkg.tar.zst") || strings.HasSuffix(file, ".pkg.tar.zst.sig") { 167 - pkg, err := arch_service.GetPackageFile(ctx, distro, file, ctx.Package.Owner.ID) 171 + if archPkgOrSig.MatchString(file) { 172 + pkg, err := arch_service.GetPackageFile(ctx, group, file, ctx.Package.Owner.ID) 168 173 if err != nil { 169 174 if errors.Is(err, util.ErrNotExist) { 170 175 apiError(ctx, http.StatusNotFound, err) ··· 180 185 return 181 186 } 182 187 183 - if strings.HasSuffix(file, ".db.tar.gz") || 184 - strings.HasSuffix(file, ".db") || 185 - strings.HasSuffix(file, ".db.tar.gz.sig") || 186 - strings.HasSuffix(file, ".db.sig") { 187 - pkg, err := arch_service.GetPackageDBFile(ctx, distro, arch, ctx.Package.Owner.ID, 188 + if archDBOrSig.MatchString(file) { 189 + pkg, err := arch_service.GetPackageDBFile(ctx, group, arch, ctx.Package.Owner.ID, 188 190 strings.HasSuffix(file, ".sig")) 189 191 if err != nil { 190 192 if errors.Is(err, util.ErrNotExist) { ··· 205 207 206 208 func RemovePackage(ctx *context.Context) { 207 209 var ( 208 - distro = ctx.Params("distro") 209 - pkg = ctx.Params("package") 210 - ver = ctx.Params("version") 210 + group = ctx.Params("group") 211 + pkg = ctx.Params("package") 212 + ver = ctx.Params("version") 211 213 ) 212 214 pv, err := packages_model.GetVersionByNameAndVersion( 213 215 ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg, ver, ··· 227 229 } 228 230 deleted := false 229 231 for _, file := range files { 230 - if file.CompositeKey == distro { 232 + if file.CompositeKey == group { 231 233 deleted = true 232 234 err := packages_service.RemovePackageFileAndVersionIfUnreferenced(ctx, ctx.ContextUser, file) 233 235 if err != nil { ··· 237 239 } 238 240 } 239 241 if deleted { 240 - err = arch_service.BuildCustomRepositoryFiles(ctx, ctx.Package.Owner.ID, distro) 242 + err = arch_service.BuildCustomRepositoryFiles(ctx, ctx.Package.Owner.ID, group) 241 243 if err != nil { 242 244 apiError(ctx, http.StatusInternalServerError, err) 243 245 }

+1 -1

services/forms/package_form.go

··· 15 15 type PackageCleanupRuleForm struct { 16 16 ID int64 17 17 Enabled bool 18 - Type string `binding:"Required;In(alpine,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"` 18 + Type string `binding:"Required;In(alpine,arch,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"` 19 19 KeepCount int `binding:"In(0,1,5,10,25,50,100)"` 20 20 KeepPattern string `binding:"RegexPattern"` 21 21 RemoveDays int `binding:"In(0,7,14,30,60,90,180)"`

+37 -24

services/packages/arch/repository.go

··· 11 11 "fmt" 12 12 "io" 13 13 "os" 14 + "path/filepath" 14 15 "sort" 15 16 "strings" 16 17 ··· 43 44 } 44 45 for _, pf := range pfs { 45 46 if strings.HasSuffix(pf.Name, ".db") { 46 - arch := strings.TrimSuffix(strings.TrimPrefix(pf.Name, fmt.Sprintf("%s-", pf.CompositeKey)), ".db") 47 + arch := strings.TrimSuffix(pf.Name, ".db") 47 48 if err := BuildPacmanDB(ctx, ownerID, pf.CompositeKey, arch); err != nil { 48 49 return err 49 50 } ··· 99 100 } 100 101 101 102 // BuildPacmanDB Create db signature cache 102 - func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) error { 103 + func BuildPacmanDB(ctx context.Context, ownerID int64, group, arch string) error { 103 104 pv, err := GetOrCreateRepositoryVersion(ctx, ownerID) 104 105 if err != nil { 105 106 return err ··· 110 111 return err 111 112 } 112 113 for _, pf := range pfs { 113 - if pf.CompositeKey == distro && strings.HasPrefix(pf.Name, fmt.Sprintf("%s-%s", distro, arch)) { 114 - // remove distro and arch 114 + if pf.CompositeKey == group && pf.Name == fmt.Sprintf("%s.db", arch) { 115 + // remove group and arch 115 116 if err := packages_service.DeletePackageFile(ctx, pf); err != nil { 116 117 return err 117 118 } 118 119 } 119 120 } 120 121 121 - db, err := flushDB(ctx, ownerID, distro, arch) 122 + db, err := createDB(ctx, ownerID, group, arch) 122 123 if errors.Is(err, io.EOF) { 123 124 return nil 124 125 } else if err != nil { ··· 140 141 return err 141 142 } 142 143 for name, data := range map[string]*packages_module.HashedBuffer{ 143 - fmt.Sprintf("%s-%s.db", distro, arch): db, 144 - fmt.Sprintf("%s-%s.db.sig", distro, arch): sig, 144 + fmt.Sprintf("%s.db", arch): db, 145 + fmt.Sprintf("%s.db.sig", arch): sig, 145 146 } { 146 147 _, err = packages_service.AddFileToPackageVersionInternal(ctx, pv, &packages_service.PackageFileCreationInfo{ 147 148 PackageFileInfo: packages_service.PackageFileInfo{ 148 149 Filename: name, 149 - CompositeKey: distro, 150 + CompositeKey: group, 150 151 }, 151 152 Creator: user_model.NewGhostUser(), 152 153 Data: data, ··· 160 161 return nil 161 162 } 162 163 163 - func flushDB(ctx context.Context, ownerID int64, distro, arch string) (*packages_module.HashedBuffer, error) { 164 + func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages_module.HashedBuffer, error) { 164 165 pkgs, err := packages_model.GetPackagesByType(ctx, ownerID, packages_model.TypeArch) 165 166 if err != nil { 166 167 return nil, err ··· 185 186 sort.Slice(versions, func(i, j int) bool { 186 187 return versions[i].CreatedUnix > versions[j].CreatedUnix 187 188 }) 189 + 188 190 for _, ver := range versions { 189 - file := fmt.Sprintf("%s-%s-%s.pkg.tar.zst", pkg.Name, ver.Version, arch) 190 - pf, err := packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro) 191 + files, err := packages_model.GetFilesByVersionID(ctx, ver.ID) 191 192 if err != nil { 192 - // add any arch package 193 - file = fmt.Sprintf("%s-%s-any.pkg.tar.zst", pkg.Name, ver.Version) 194 - pf, err = packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro) 195 - if err != nil { 196 - continue 193 + return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) 194 + } 195 + var pf *packages_model.PackageFile 196 + for _, file := range files { 197 + ext := filepath.Ext(file.Name) 198 + if file.CompositeKey == group && ext != "" && ext != ".db" && ext != ".sig" { 199 + if pf == nil && strings.HasSuffix(file.Name, fmt.Sprintf("any.pkg.tar%s", ext)) { 200 + pf = file 201 + } 202 + if strings.HasSuffix(file.Name, fmt.Sprintf("%s.pkg.tar%s", arch, ext)) { 203 + pf = file 204 + break 205 + } 197 206 } 207 + } 208 + if pf == nil { 209 + // file not exists 210 + continue 198 211 } 199 212 pps, err := packages_model.GetPropertiesByName( 200 213 ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyDescription, ··· 230 243 231 244 // GetPackageFile Get data related to provided filename and distribution, for package files 232 245 // update download counter. 233 - func GetPackageFile(ctx context.Context, distro, file string, ownerID int64) (io.ReadSeekCloser, error) { 234 - pf, err := getPackageFile(ctx, distro, file, ownerID) 246 + func GetPackageFile(ctx context.Context, group, file string, ownerID int64) (io.ReadSeekCloser, error) { 247 + pf, err := getPackageFile(ctx, group, file, ownerID) 235 248 if err != nil { 236 249 return nil, err 237 250 } ··· 241 254 } 242 255 243 256 // Ejects parameters required to get package file property from file name. 244 - func getPackageFile(ctx context.Context, distro, file string, ownerID int64) (*packages_model.PackageFile, error) { 257 + func getPackageFile(ctx context.Context, group, file string, ownerID int64) (*packages_model.PackageFile, error) { 245 258 var ( 246 259 splt = strings.Split(file, "-") 247 260 pkgname = strings.Join(splt[0:len(splt)-3], "-") ··· 253 266 return nil, err 254 267 } 255 268 256 - pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, distro) 269 + pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, group) 257 270 if err != nil { 258 271 return nil, err 259 272 } 260 273 return pkgfile, nil 261 274 } 262 275 263 - func GetPackageDBFile(ctx context.Context, distro, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, error) { 276 + func GetPackageDBFile(ctx context.Context, group, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, error) { 264 277 pv, err := GetOrCreateRepositoryVersion(ctx, ownerID) 265 278 if err != nil { 266 279 return nil, err 267 280 } 268 - fileName := fmt.Sprintf("%s-%s.db", distro, arch) 281 + fileName := fmt.Sprintf("%s.db", arch) 269 282 if signFile { 270 - fileName = fmt.Sprintf("%s-%s.db.sig", distro, arch) 283 + fileName = fmt.Sprintf("%s.db.sig", arch) 271 284 } 272 - file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, distro) 285 + file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, group) 273 286 if err != nil { 274 287 return nil, err 275 288 }

+137 -138

tests/integration/api_packages_arch_test.go

··· 81 81 KNlA0hEjIOACGSLqYpXAD5SSNVT2MJRJwREAF4FRHPBlCJMSNwFguGAWDJBg+KIArkIJGNtCydUL 82 82 TuN1oBh/+zKkEblAsgjGqVgUwKLP+UOMOGCpAhICtg6ncFJH`), 83 83 "other": unPack(` 84 - KLUv/QBYbRMABuOHS9BSNQdQ56F+xNFoV3CijY54JYt3VqV1iUU3xmj00y2pyBOCuokbhDYpvNsj 85 - ZJeCxqH+nQFpMf4Wa92okaZoF4eH6HsXXCBo+qy3Fn4AigBgAEaYrLCQEuAom6YbHyuKZAFYksqi 86 - sSOFiRs0WDmlACk0CnpnaAeKiCS3BlwVkViJEbDS43lFNbLkZEmGhc305Nn4AMLGiUkBDiMTG5Vz 87 - q4ZISjCofEfR1NpXijvP2X95Hu1e+zLalc0+mjeT3Z/FPGvt62WymbX2dXMDIYKDLjjP8n03RrPf 88 - A1vOApwGOh2MgE2LpgZrgXLDF2CUJ15idG2J8GCSgcc2ZVRgA8+RHD0k2VJjg6mRUgGGhBWEyEcz 89 - 5EePLhUeWlYhoFCKONxUiBiIUiQeDIqiQwkjLiyqnF5eGs6a2gGRapbU9JRyuXAlPemYajlJojJd 90 - GBBJjo5GxFRkITOAvLhSCr2TDz4uzdU8Yh3i/SHP4qh3vTG2s9198NP8M+pdR73BvIP6qPeDjzsW 91 - gTi+jXrXWOe5P/jZxOeod/287v6JljzNP99RNM0a+/x4ljz3LNV2t5v9qHfW2Pyg24u54zSfObWX 92 - Y9bYrCTHtwdfPPPOYiU5fvB5FssfNN2V5EIPfg9LnM+JhtVEO8+FZw5LXA068YNPhimu9sHPQiWv 93 - qc6fE9BTnxIe/LTKatab+WYu7T74uWNRxJW5W5Ux0bDLuG1ioCwjg4DvGgBcgB8cUDHJ1RQ89neE 94 - wvjbNUMiIZdo5hbHgEpANwMkDnL0Jr7kVFg+0pZKjBkmklNgBH1YI8dQOAAKbr6EF5wYM80KWnAd 95 - nYAR`), 84 + /Td6WFoAAATm1rRGBMCyBIAYIQEWAAAAAAAAABaHRszgC/8CKl0AFxNGhTWwfXmuDQEJlHgNLrkq 85 + VxpJY6d9iRTt6gB4uCj0481rnYfXaUADHzOFuF3490RPrM6juPXrknqtVyuWJ5efW19BgwctN6xk 86 + UiXiZaXVAWVWJWy2XHJiyYCMWBfIjUfo1ccOgwolwgFHJ64ZJjbayA3k6lYPcImuAqYL5NEVHpwl 87 + Z8CWIjiXXSMQGsB3gxMdq9nySZbHQLK/KCKQ+oseF6kXyIgSEyuG4HhjVBBYIwTvWzI06kjNUXEy 88 + 2sw0n50uocLSAwJ/3mdX3n3XF5nmmuQMPtFbdQgQtC2VhyVd3TdIF+pT6zAEzXFJJ3uLkNbKSS88 89 + ZdBny6X/ftT5lQpNi/Wg0xLEQA4m4fu4fRAR0kOKzHM2svNLbTxa/wOPidqPzR6b/jfKmHkXxBNa 90 + jFafty0a5K2S3F6JpwXZ2fqti/zG9NtMc+bbuXycC327EofXRXNtuOupELDD+ltTOIBF7CcTswyi 91 + MZDP1PBie6GqDV2GuPz+0XXmul/ds+XysG19HIkKbJ+cQKp5o7Y0tI7EHM8GhwMl7MjgpQGj5nuv 92 + 0u2hqt4NXPNYqaMm9bFnnIUxEN82HgNWBcXf2baWKOdGzPzCuWg2fAM4zxHnBWcimxLXiJgaI8mU 93 + J/QqTPWE0nJf1PW/J9yFQVR1Xo0TJyiX8/ObwmbqUPpxRGjKlYRBvn0jbTdUAENBSn+QVcASRGFE 94 + SB9OM2B8Bg4jR/oojs8Beoq7zbIblgAAAACfRtXvhmznOgABzgSAGAAAKklb4rHEZ/sCAAAAAARZ 95 + Wg==`), // this is tar.xz file 96 96 } 97 97 98 98 t.Run("RepositoryKey", func(t *testing.T) { ··· 105 105 require.Contains(t, resp.Body.String(), "-----BEGIN PGP PUBLIC KEY BLOCK-----") 106 106 }) 107 107 108 - t.Run("Upload", func(t *testing.T) { 109 - defer tests.PrintCurrentTest(t)() 108 + for _, group := range []string{"", "arch", "arch/os", "x86_64"} { 109 + groupURL := rootURL 110 + if group != "" { 111 + groupURL = groupURL + "/" + group 112 + } 113 + t.Run(fmt.Sprintf("Upload[%s]", group), func(t *testing.T) { 114 + defer tests.PrintCurrentTest(t)() 110 115 111 - req := NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])) 112 - MakeRequest(t, req, http.StatusUnauthorized) 116 + req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"])) 117 + MakeRequest(t, req, http.StatusUnauthorized) 113 118 114 - req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])). 115 - AddBasicAuth(user.Name) 116 - MakeRequest(t, req, http.StatusCreated) 119 + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"])). 120 + AddBasicAuth(user.Name) 121 + MakeRequest(t, req, http.StatusCreated) 117 122 118 - pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeArch) 119 - require.NoError(t, err) 120 - require.Len(t, pvs, 1) 123 + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewBuffer([]byte("any string"))). 124 + AddBasicAuth(user.Name) 125 + MakeRequest(t, req, http.StatusBadRequest) 121 126 122 - pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) 123 - require.NoError(t, err) 124 - require.Nil(t, pd.SemVer) 125 - require.IsType(t, &arch_model.VersionMetadata{}, pd.Metadata) 126 - require.Equal(t, "test", pd.Package.Name) 127 - require.Equal(t, "1.0.0-1", pd.Version.Version) 127 + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeArch) 128 + require.NoError(t, err) 129 + require.Len(t, pvs, 1) 128 130 129 - pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) 130 - require.NoError(t, err) 131 - require.Len(t, pfs, 2) // zst and zst.sig 132 - require.True(t, pfs[0].IsLead) 131 + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) 132 + require.NoError(t, err) 133 + require.Nil(t, pd.SemVer) 134 + require.IsType(t, &arch_model.VersionMetadata{}, pd.Metadata) 135 + require.Equal(t, "test", pd.Package.Name) 136 + require.Equal(t, "1.0.0-1", pd.Version.Version) 133 137 134 - pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) 135 - require.NoError(t, err) 136 - require.Equal(t, int64(len(pkgs["any"])), pb.Size) 138 + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) 139 + require.NoError(t, err) 140 + size := 0 141 + for _, pf := range pfs { 142 + if pf.CompositeKey == group { 143 + size++ 144 + } 145 + } 146 + require.Equal(t, 2, size) // zst and zst.sig 137 147 138 - req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])). 139 - AddBasicAuth(user.Name) 140 - MakeRequest(t, req, http.StatusConflict) 141 - req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["x86_64"])). 142 - AddBasicAuth(user.Name) 143 - MakeRequest(t, req, http.StatusCreated) 144 - req = NewRequestWithBody(t, "PUT", rootURL+"/other", bytes.NewReader(pkgs["any"])). 145 - AddBasicAuth(user.Name) 146 - MakeRequest(t, req, http.StatusCreated) 147 - req = NewRequestWithBody(t, "PUT", rootURL+"/other", bytes.NewReader(pkgs["aarch64"])). 148 - AddBasicAuth(user.Name) 149 - MakeRequest(t, req, http.StatusCreated) 148 + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) 149 + require.NoError(t, err) 150 + require.Equal(t, int64(len(pkgs["any"])), pb.Size) 150 151 151 - req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["other"])). 152 - AddBasicAuth(user.Name) 153 - MakeRequest(t, req, http.StatusCreated) 154 - req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["x86_64"])). 155 - AddBasicAuth(user.Name) 156 - MakeRequest(t, req, http.StatusCreated) 157 - req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["aarch64"])). 158 - AddBasicAuth(user.Name) 159 - MakeRequest(t, req, http.StatusCreated) 160 - }) 152 + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"])). 153 + AddBasicAuth(user.Name) // exists 154 + MakeRequest(t, req, http.StatusConflict) 155 + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["x86_64"])). 156 + AddBasicAuth(user.Name) 157 + MakeRequest(t, req, http.StatusCreated) 158 + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["aarch64"])). 159 + AddBasicAuth(user.Name) 160 + MakeRequest(t, req, http.StatusCreated) 161 + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["aarch64"])). 162 + AddBasicAuth(user.Name) // exists again 163 + MakeRequest(t, req, http.StatusConflict) 164 + }) 161 165 162 - t.Run("Download", func(t *testing.T) { 163 - defer tests.PrintCurrentTest(t)() 164 - req := NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst") 165 - resp := MakeRequest(t, req, http.StatusOK) 166 - require.Equal(t, pkgs["x86_64"], resp.Body.Bytes()) 166 + t.Run(fmt.Sprintf("Download[%s]", group), func(t *testing.T) { 167 + defer tests.PrintCurrentTest(t)() 168 + req := NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst") 169 + resp := MakeRequest(t, req, http.StatusOK) 170 + require.Equal(t, pkgs["x86_64"], resp.Body.Bytes()) 167 171 168 - req = NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-any.pkg.tar.zst") 169 - resp = MakeRequest(t, req, http.StatusOK) 170 - require.Equal(t, pkgs["any"], resp.Body.Bytes()) 172 + req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst") 173 + resp = MakeRequest(t, req, http.StatusOK) 174 + require.Equal(t, pkgs["any"], resp.Body.Bytes()) 171 175 172 - req = NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-aarch64.pkg.tar.zst") 173 - MakeRequest(t, req, http.StatusNotFound) 176 + // get other group 177 + req = NewRequest(t, "GET", rootURL+"/unknown/x86_64/test-1.0.0-1-aarch64.pkg.tar.zst") 178 + MakeRequest(t, req, http.StatusNotFound) 179 + }) 174 180 175 - req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst") 176 - MakeRequest(t, req, http.StatusNotFound) 181 + t.Run(fmt.Sprintf("SignVerify[%s]", group), func(t *testing.T) { 182 + defer tests.PrintCurrentTest(t)() 183 + req := NewRequest(t, "GET", rootURL+"/repository.key") 184 + respPub := MakeRequest(t, req, http.StatusOK) 177 185 178 - req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst") 179 - resp = MakeRequest(t, req, http.StatusOK) 180 - require.Equal(t, pkgs["any"], resp.Body.Bytes()) 181 - }) 186 + req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst") 187 + respPkg := MakeRequest(t, req, http.StatusOK) 182 188 183 - t.Run("SignVerify", func(t *testing.T) { 184 - defer tests.PrintCurrentTest(t)() 185 - req := NewRequest(t, "GET", rootURL+"/repository.key") 186 - respPub := MakeRequest(t, req, http.StatusOK) 189 + req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst.sig") 190 + respSig := MakeRequest(t, req, http.StatusOK) 187 191 188 - req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst") 189 - respPkg := MakeRequest(t, req, http.StatusOK) 192 + if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { 193 + t.Fatal(err) 194 + } 195 + }) 190 196 191 - req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst.sig") 192 - respSig := MakeRequest(t, req, http.StatusOK) 197 + t.Run(fmt.Sprintf("RepositoryDB[%s]", group), func(t *testing.T) { 198 + defer tests.PrintCurrentTest(t)() 199 + req := NewRequest(t, "GET", rootURL+"/repository.key") 200 + respPub := MakeRequest(t, req, http.StatusOK) 193 201 194 - if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { 195 - t.Fatal(err) 196 - } 197 - }) 202 + req = NewRequest(t, "GET", groupURL+"/x86_64/base.db") 203 + respPkg := MakeRequest(t, req, http.StatusOK) 198 204 199 - t.Run("Repository", func(t *testing.T) { 200 - defer tests.PrintCurrentTest(t)() 201 - req := NewRequest(t, "GET", rootURL+"/repository.key") 202 - respPub := MakeRequest(t, req, http.StatusOK) 205 + req = NewRequest(t, "GET", groupURL+"/x86_64/base.db.sig") 206 + respSig := MakeRequest(t, req, http.StatusOK) 203 207 204 - req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db") 205 - respPkg := MakeRequest(t, req, http.StatusOK) 208 + if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { 209 + t.Fatal(err) 210 + } 211 + files, err := listTarGzFiles(respPkg.Body.Bytes()) 212 + require.NoError(t, err) 213 + require.Len(t, files, 1) 214 + for s, d := range files { 215 + name := getProperty(string(d.Data), "NAME") 216 + ver := getProperty(string(d.Data), "VERSION") 217 + require.Equal(t, name+"-"+ver+"/desc", s) 218 + fn := getProperty(string(d.Data), "FILENAME") 219 + pgp := getProperty(string(d.Data), "PGPSIG") 220 + req = NewRequest(t, "GET", groupURL+"/x86_64/"+fn+".sig") 221 + respSig := MakeRequest(t, req, http.StatusOK) 222 + decodeString, err := base64.StdEncoding.DecodeString(pgp) 223 + require.NoError(t, err) 224 + require.Equal(t, respSig.Body.Bytes(), decodeString) 225 + } 226 + }) 206 227 207 - req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db.sig") 208 - respSig := MakeRequest(t, req, http.StatusOK) 209 - 210 - if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { 211 - t.Fatal(err) 212 - } 213 - files, err := listGzipFiles(respPkg.Body.Bytes()) 214 - require.NoError(t, err) 215 - require.Len(t, files, 2) 216 - for s, d := range files { 217 - name := getProperty(string(d.Data), "NAME") 218 - ver := getProperty(string(d.Data), "VERSION") 219 - require.Equal(t, name+"-"+ver+"/desc", s) 220 - fn := getProperty(string(d.Data), "FILENAME") 221 - pgp := getProperty(string(d.Data), "PGPSIG") 222 - req = NewRequest(t, "GET", rootURL+"/base/x86_64/"+fn+".sig") 223 - respSig := MakeRequest(t, req, http.StatusOK) 224 - decodeString, err := base64.StdEncoding.DecodeString(pgp) 225 - require.NoError(t, err) 226 - require.Equal(t, respSig.Body.Bytes(), decodeString) 227 - } 228 - }) 229 - t.Run("Delete", func(t *testing.T) { 230 - defer tests.PrintCurrentTest(t)() 231 - req := NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil). 232 - AddBasicAuth(user.Name) 233 - MakeRequest(t, req, http.StatusNotFound) 228 + t.Run(fmt.Sprintf("Delete[%s]", group), func(t *testing.T) { 229 + defer tests.PrintCurrentTest(t)() 230 + // test data 231 + req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["other"])). 232 + AddBasicAuth(user.Name) 233 + MakeRequest(t, req, http.StatusCreated) 234 234 235 - req = NewRequestWithBody(t, "DELETE", rootURL+"/base/test/1.0.0-1", nil). 236 - AddBasicAuth(user.Name) 237 - MakeRequest(t, req, http.StatusNoContent) 235 + req = NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil). 236 + AddBasicAuth(user.Name) 237 + MakeRequest(t, req, http.StatusNotFound) 238 238 239 - req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db") 240 - respPkg := MakeRequest(t, req, http.StatusOK) 241 - files, err := listGzipFiles(respPkg.Body.Bytes()) 242 - require.NoError(t, err) 243 - require.Len(t, files, 1) 239 + req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1", nil). 240 + AddBasicAuth(user.Name) 241 + MakeRequest(t, req, http.StatusNoContent) 244 242 245 - req = NewRequestWithBody(t, "DELETE", rootURL+"/base/test2/1.0.0-1", nil). 246 - AddBasicAuth(user.Name) 247 - MakeRequest(t, req, http.StatusNoContent) 248 - req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db") 249 - MakeRequest(t, req, http.StatusNotFound) 243 + req = NewRequest(t, "GET", groupURL+"/x86_64/base.db") 244 + respPkg := MakeRequest(t, req, http.StatusOK) 245 + files, err := listTarGzFiles(respPkg.Body.Bytes()) 246 + require.NoError(t, err) 247 + require.Len(t, files, 1) // other pkg in L225 250 248 251 - req = NewRequest(t, "GET", rootURL+"/default/x86_64/base.db") 252 - respPkg = MakeRequest(t, req, http.StatusOK) 253 - files, err = listGzipFiles(respPkg.Body.Bytes()) 254 - require.NoError(t, err) 255 - require.Len(t, files, 1) 256 - }) 249 + req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil). 250 + AddBasicAuth(user.Name) 251 + MakeRequest(t, req, http.StatusNoContent) 252 + req = NewRequest(t, "GET", groupURL+"/x86_64/base.db") 253 + MakeRequest(t, req, http.StatusNotFound) 254 + }) 255 + } 257 256 } 258 257 259 258 func getProperty(data, key string) string { ··· 270 269 } 271 270 } 272 271 273 - func listGzipFiles(data []byte) (fstest.MapFS, error) { 272 + func listTarGzFiles(data []byte) (fstest.MapFS, error) { 274 273 reader, err := gzip.NewReader(bytes.NewBuffer(data)) 275 274 defer reader.Close() 276 275 if err != nil {

Configure Feed

Configure Feed