+1 -1

cmd/serv.go

reviewed

··· 17 17 "time" 18 18 19 19 "code.gitea.io/gitea/models" 20 20 - "code.gitea.io/gitea/modules/lfs" 21 20 "code.gitea.io/gitea/modules/log" 22 21 "code.gitea.io/gitea/modules/pprof" 23 22 "code.gitea.io/gitea/modules/private" 24 23 "code.gitea.io/gitea/modules/setting" 24 24 + "code.gitea.io/gitea/services/lfs" 25 25 26 26 "github.com/dgrijalva/jwt-go" 27 27 jsoniter "github.com/json-iterator/go"

+49

integrations/api_repo_lfs_migrate_test.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package integrations 6 6 + 7 7 + import ( 8 8 + "net/http" 9 9 + "path" 10 10 + "testing" 11 11 + 12 12 + "code.gitea.io/gitea/models" 13 13 + "code.gitea.io/gitea/modules/lfs" 14 14 + "code.gitea.io/gitea/modules/setting" 15 15 + api "code.gitea.io/gitea/modules/structs" 16 16 + 17 17 + "github.com/stretchr/testify/assert" 18 18 + ) 19 19 + 20 20 + func TestAPIRepoLFSMigrateLocal(t *testing.T) { 21 21 + defer prepareTestEnv(t)() 22 22 + 23 23 + oldImportLocalPaths := setting.ImportLocalPaths 24 24 + oldAllowLocalNetworks := setting.Migrations.AllowLocalNetworks 25 25 + setting.ImportLocalPaths = true 26 26 + setting.Migrations.AllowLocalNetworks = true 27 27 + 28 28 + user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User) 29 29 + session := loginUser(t, user.Name) 30 30 + token := getTokenForLoggedInUser(t, session) 31 31 + 32 32 + req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate?token="+token, &api.MigrateRepoOptions{ 33 33 + CloneAddr: path.Join(setting.RepoRootPath, "migration/lfs-test.git"), 34 34 + RepoOwnerID: user.ID, 35 35 + RepoName: "lfs-test-local", 36 36 + LFS: true, 37 37 + }) 38 38 + resp := MakeRequest(t, req, NoExpectedStatus) 39 39 + assert.EqualValues(t, http.StatusCreated, resp.Code) 40 40 + 41 41 + store := lfs.NewContentStore() 42 42 + ok, _ := store.Verify(lfs.Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6}) 43 43 + assert.True(t, ok) 44 44 + ok, _ = store.Verify(lfs.Pointer{Oid: "d6f175817f886ec6fbbc1515326465fa96c3bfd54a4ea06cfd6dbbd8340e0152", Size: 6}) 45 45 + assert.True(t, ok) 46 46 + 47 47 + setting.ImportLocalPaths = oldImportLocalPaths 48 48 + setting.Migrations.AllowLocalNetworks = oldAllowLocalNetworks 49 49 + }

+3 -2

integrations/git_test.go

reviewed

··· 18 18 19 19 "code.gitea.io/gitea/models" 20 20 "code.gitea.io/gitea/modules/git" 21 21 + "code.gitea.io/gitea/modules/lfs" 21 22 "code.gitea.io/gitea/modules/setting" 22 23 api "code.gitea.io/gitea/modules/structs" 23 24 "code.gitea.io/gitea/modules/util" ··· 218 219 assert.NotEqual(t, littleSize, resp.Body.Len()) 219 220 assert.LessOrEqual(t, resp.Body.Len(), 1024) 220 221 if resp.Body.Len() != littleSize && resp.Body.Len() <= 1024 { 221 221 - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) 222 222 + assert.Contains(t, resp.Body.String(), lfs.MetaFileIdentifier) 222 223 } 223 224 } 224 225 ··· 232 233 resp := session.MakeRequest(t, req, http.StatusOK) 233 234 assert.NotEqual(t, bigSize, resp.Body.Len()) 234 235 if resp.Body.Len() != bigSize && resp.Body.Len() <= 1024 { 235 235 - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) 236 236 + assert.Contains(t, resp.Body.String(), lfs.MetaFileIdentifier) 236 237 } 237 238 } 238 239 }

+1

integrations/gitea-repositories-meta/migration/lfs-test.git/HEAD

reviewed

··· 1 1 + ref: refs/heads/master

+7

integrations/gitea-repositories-meta/migration/lfs-test.git/config

reviewed

··· 1 1 + [core] 2 2 + bare = false 3 3 + repositoryformatversion = 0 4 4 + filemode = false 5 5 + symlinks = false 6 6 + ignorecase = true 7 7 + logallrefupdates = true

+1

integrations/gitea-repositories-meta/migration/lfs-test.git/description

reviewed

··· 1 1 + Unnamed repository; edit this file 'description' to name the repository.

+3

integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-checkout

reviewed

··· 1 1 + #!/bin/sh 2 2 + command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout.\n"; exit 2; } 3 3 + git lfs post-checkout "$@"

+3

integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-commit

reviewed

··· 1 1 + #!/bin/sh 2 2 + command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-commit.\n"; exit 2; } 3 3 + git lfs post-commit "$@"

+3

integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-merge

reviewed

··· 1 1 + #!/bin/sh 2 2 + command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-merge.\n"; exit 2; } 3 3 + git lfs post-merge "$@"

+3

integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/pre-push

reviewed

··· 1 1 + #!/bin/sh 2 2 + command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/pre-push.\n"; exit 2; } 3 3 + git lfs pre-push "$@"

integrations/gitea-repositories-meta/migration/lfs-test.git/index

reviewed

This is a binary file and will not be displayed.

+1

integrations/gitea-repositories-meta/migration/lfs-test.git/lfs/objects/d6/f1/d6f175817f886ec6fbbc1515326465fa96c3bfd54a4ea06cfd6dbbd8340e0152

reviewed

··· 1 1 + dummy2

+1

integrations/gitea-repositories-meta/migration/lfs-test.git/lfs/objects/fb/8f/fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041

reviewed

··· 1 1 + dummy1

integrations/gitea-repositories-meta/migration/lfs-test.git/objects/54/6244003622c64b2fc3c2cd544d7a29882c8383

reviewed

This is a binary file and will not be displayed.

integrations/gitea-repositories-meta/migration/lfs-test.git/objects/6a/6ccf5d874fec134ee712572cc03a0f2dd7afec

reviewed

This is a binary file and will not be displayed.

integrations/gitea-repositories-meta/migration/lfs-test.git/objects/a6/7134b8484c2abe9fa954e1fd83b39b271383ed

reviewed

This is a binary file and will not be displayed.

integrations/gitea-repositories-meta/migration/lfs-test.git/objects/b7/01ed6ffe410f0c3ac204b929ea47cfec6cef54

reviewed

This is a binary file and will not be displayed.

integrations/gitea-repositories-meta/migration/lfs-test.git/objects/f2/07b74f55cd7f9e800b7550d587cbc488f6eaf1

reviewed

This is a binary file and will not be displayed.

+1

integrations/gitea-repositories-meta/migration/lfs-test.git/refs/heads/master

reviewed

··· 1 1 + 546244003622c64b2fc3c2cd544d7a29882c8383

+7 -20

integrations/lfs_getobject_test.go

reviewed

··· 7 7 import ( 8 8 "archive/zip" 9 9 "bytes" 10 10 - "crypto/sha256" 11 11 - "encoding/hex" 12 12 - "io" 13 10 "io/ioutil" 14 11 "net/http" 15 12 "net/http/httptest" ··· 18 15 "code.gitea.io/gitea/models" 19 16 "code.gitea.io/gitea/modules/lfs" 20 17 "code.gitea.io/gitea/modules/setting" 21 21 - "code.gitea.io/gitea/modules/storage" 22 18 "code.gitea.io/gitea/routers/routes" 23 19 24 20 gzipp "github.com/klauspost/compress/gzip" 25 21 "github.com/stretchr/testify/assert" 26 22 ) 27 23 28 28 - func GenerateLFSOid(content io.Reader) (string, error) { 29 29 - h := sha256.New() 30 30 - if _, err := io.Copy(h, content); err != nil { 31 31 - return "", err 32 32 - } 33 33 - sum := h.Sum(nil) 34 34 - return hex.EncodeToString(sum), nil 35 35 - } 36 36 - 37 24 var lfsID = int64(20000) 38 25 39 26 func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string { 40 40 - oid, err := GenerateLFSOid(bytes.NewReader(*content)) 27 27 + pointer, err := lfs.GeneratePointer(bytes.NewReader(*content)) 41 28 assert.NoError(t, err) 42 29 var lfsMetaObject *models.LFSMetaObject 43 30 44 31 if setting.Database.UsePostgreSQL { 45 45 - lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID} 32 32 + lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Pointer: pointer, RepositoryID: repositoryID} 46 33 } else { 47 47 - lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID} 34 34 + lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: repositoryID} 48 35 } 49 36 50 37 lfsID++ 51 38 lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) 52 39 assert.NoError(t, err) 53 53 - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} 54 54 - exist, err := contentStore.Exists(lfsMetaObject) 40 40 + contentStore := lfs.NewContentStore() 41 41 + exist, err := contentStore.Exists(pointer) 55 42 assert.NoError(t, err) 56 43 if !exist { 57 57 - err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content)) 44 44 + err := contentStore.Put(pointer, bytes.NewReader(*content)) 58 45 assert.NoError(t, err) 59 46 } 60 60 - return oid 47 47 + return pointer.Oid 61 48 } 62 49 63 50 func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, expectedStatus int) *httptest.ResponseRecorder {

+117

integrations/lfs_local_endpoint_test.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package integrations 6 6 + 7 7 + import ( 8 8 + "fmt" 9 9 + "io/ioutil" 10 10 + "net/url" 11 11 + "os" 12 12 + "path/filepath" 13 13 + "testing" 14 14 + 15 15 + "code.gitea.io/gitea/modules/lfs" 16 16 + 17 17 + "github.com/stretchr/testify/assert" 18 18 + ) 19 19 + 20 20 + func str2url(raw string) *url.URL { 21 21 + u, _ := url.Parse(raw) 22 22 + return u 23 23 + } 24 24 + 25 25 + func TestDetermineLocalEndpoint(t *testing.T) { 26 26 + defer prepareTestEnv(t)() 27 27 + 28 28 + root, _ := ioutil.TempDir("", "lfs_test") 29 29 + defer os.RemoveAll(root) 30 30 + 31 31 + rootdotgit, _ := ioutil.TempDir("", "lfs_test") 32 32 + defer os.RemoveAll(rootdotgit) 33 33 + os.Mkdir(filepath.Join(rootdotgit, ".git"), 0700) 34 34 + 35 35 + lfsroot, _ := ioutil.TempDir("", "lfs_test") 36 36 + defer os.RemoveAll(lfsroot) 37 37 + 38 38 + // Test cases 39 39 + var cases = []struct { 40 40 + cloneurl string 41 41 + lfsurl string 42 42 + expected *url.URL 43 43 + }{ 44 44 + // case 0 45 45 + { 46 46 + cloneurl: root, 47 47 + lfsurl: "", 48 48 + expected: str2url(fmt.Sprintf("file://%s", root)), 49 49 + }, 50 50 + // case 1 51 51 + { 52 52 + cloneurl: root, 53 53 + lfsurl: lfsroot, 54 54 + expected: str2url(fmt.Sprintf("file://%s", lfsroot)), 55 55 + }, 56 56 + // case 2 57 57 + { 58 58 + cloneurl: "https://git.com/repo.git", 59 59 + lfsurl: lfsroot, 60 60 + expected: str2url(fmt.Sprintf("file://%s", lfsroot)), 61 61 + }, 62 62 + // case 3 63 63 + { 64 64 + cloneurl: rootdotgit, 65 65 + lfsurl: "", 66 66 + expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))), 67 67 + }, 68 68 + // case 4 69 69 + { 70 70 + cloneurl: "", 71 71 + lfsurl: rootdotgit, 72 72 + expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))), 73 73 + }, 74 74 + // case 5 75 75 + { 76 76 + cloneurl: rootdotgit, 77 77 + lfsurl: rootdotgit, 78 78 + expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))), 79 79 + }, 80 80 + // case 6 81 81 + { 82 82 + cloneurl: fmt.Sprintf("file://%s", root), 83 83 + lfsurl: "", 84 84 + expected: str2url(fmt.Sprintf("file://%s", root)), 85 85 + }, 86 86 + // case 7 87 87 + { 88 88 + cloneurl: fmt.Sprintf("file://%s", root), 89 89 + lfsurl: fmt.Sprintf("file://%s", lfsroot), 90 90 + expected: str2url(fmt.Sprintf("file://%s", lfsroot)), 91 91 + }, 92 92 + // case 8 93 93 + { 94 94 + cloneurl: root, 95 95 + lfsurl: fmt.Sprintf("file://%s", lfsroot), 96 96 + expected: str2url(fmt.Sprintf("file://%s", lfsroot)), 97 97 + }, 98 98 + // case 9 99 99 + { 100 100 + cloneurl: "", 101 101 + lfsurl: "/does/not/exist", 102 102 + expected: nil, 103 103 + }, 104 104 + // case 10 105 105 + { 106 106 + cloneurl: "", 107 107 + lfsurl: "file:///does/not/exist", 108 108 + expected: str2url("file:///does/not/exist"), 109 109 + }, 110 110 + } 111 111 + 112 112 + for n, c := range cases { 113 113 + ep := lfs.DetermineEndpoint(c.cloneurl, c.lfsurl) 114 114 + 115 115 + assert.Equal(t, c.expected, ep, "case %d: error should match", n) 116 116 + } 117 117 + }

+8 -46

models/lfs.go

reviewed

··· 5 5 package models 6 6 7 7 import ( 8 8 - "crypto/sha256" 9 9 - "encoding/hex" 10 8 "errors" 11 11 - "fmt" 12 12 - "io" 13 13 - "path" 14 9 10 10 + "code.gitea.io/gitea/modules/lfs" 15 11 "code.gitea.io/gitea/modules/timeutil" 16 12 17 13 "xorm.io/builder" ··· 19 15 20 16 // LFSMetaObject stores metadata for LFS tracked files. 21 17 type LFSMetaObject struct { 22 22 - ID int64 `xorm:"pk autoincr"` 23 23 - Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"` 24 24 - Size int64 `xorm:"NOT NULL"` 18 18 + ID int64 `xorm:"pk autoincr"` 19 19 + lfs.Pointer `xorm:"extends"` 25 20 RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` 26 21 Existing bool `xorm:"-"` 27 22 CreatedUnix timeutil.TimeStamp `xorm:"created"` 28 23 } 29 24 30 30 - // RelativePath returns the relative path of the lfs object 31 31 - func (m *LFSMetaObject) RelativePath() string { 32 32 - if len(m.Oid) < 5 { 33 33 - return m.Oid 34 34 - } 35 35 - 36 36 - return path.Join(m.Oid[0:2], m.Oid[2:4], m.Oid[4:]) 37 37 - } 38 38 - 39 39 - // Pointer returns the string representation of an LFS pointer file 40 40 - func (m *LFSMetaObject) Pointer() string { 41 41 - return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size) 42 42 - } 43 43 - 44 25 // LFSTokenResponse defines the JSON structure in which the JWT token is stored. 45 26 // This structure is fetched via SSH and passed by the Git LFS client to the server 46 27 // endpoint for authorization. ··· 53 34 // to differentiate between database and missing object errors. 54 35 var ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist") 55 36 56 56 - const ( 57 57 - // LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files. 58 58 - // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md 59 59 - LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" 60 60 - 61 61 - // LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. 62 62 - LFSMetaFileOidPrefix = "oid sha256:" 63 63 - ) 64 64 - 65 37 // NewLFSMetaObject stores a given populated LFSMetaObject structure in the database 66 38 // if it is not already present. 67 39 func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { ··· 90 62 return m, sess.Commit() 91 63 } 92 64 93 93 - // GenerateLFSOid generates a Sha256Sum to represent an oid for arbitrary content 94 94 - func GenerateLFSOid(content io.Reader) (string, error) { 95 95 - h := sha256.New() 96 96 - if _, err := io.Copy(h, content); err != nil { 97 97 - return "", err 98 98 - } 99 99 - sum := h.Sum(nil) 100 100 - return hex.EncodeToString(sum), nil 101 101 - } 102 102 - 103 65 // GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID. 104 66 // It may return ErrLFSObjectNotExist or a database error. If the error is nil, 105 67 // the returned pointer is a valid LFSMetaObject. ··· 108 70 return nil, ErrLFSObjectNotExist 109 71 } 110 72 111 111 - m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID} 73 73 + m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repo.ID} 112 74 has, err := x.Get(m) 113 75 if err != nil { 114 76 return nil, err ··· 131 93 return -1, err 132 94 } 133 95 134 134 - m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID} 96 96 + m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repo.ID} 135 97 if _, err := sess.Delete(m); err != nil { 136 98 return -1, err 137 99 } 138 100 139 139 - count, err := sess.Count(&LFSMetaObject{Oid: oid}) 101 101 + count, err := sess.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) 140 102 if err != nil { 141 103 return count, err 142 104 } ··· 168 130 // LFSObjectAccessible checks if a provided Oid is accessible to the user 169 131 func LFSObjectAccessible(user *User, oid string) (bool, error) { 170 132 if user.IsAdmin { 171 171 - count, err := x.Count(&LFSMetaObject{Oid: oid}) 133 133 + count, err := x.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) 172 134 return (count > 0), err 173 135 } 174 136 cond := accessibleRepositoryCondition(user) 175 175 - count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Oid: oid}) 137 137 + count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) 176 138 return (count > 0), err 177 139 } 178 140

+2

models/migrations/migrations.go

reviewed

··· 302 302 NewMigration("Remove invalid labels from comments", removeInvalidLabels), 303 303 // v177 -> v178 304 304 NewMigration("Delete orphaned IssueLabels", deleteOrphanedIssueLabels), 305 305 + // v178 -> v179 306 306 + NewMigration("Add LFS columns to Mirror", addLFSMirrorColumns), 305 307 } 306 308 307 309 // GetCurrentDBVersion returns the current db version

+18

models/migrations/v178.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package migrations 6 6 + 7 7 + import ( 8 8 + "xorm.io/xorm" 9 9 + ) 10 10 + 11 11 + func addLFSMirrorColumns(x *xorm.Engine) error { 12 12 + type Mirror struct { 13 13 + LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"` 14 14 + LFSEndpoint string `xorm:"lfs_endpoint TEXT"` 15 15 + } 16 16 + 17 17 + return x.Sync2(new(Mirror)) 18 18 + }

+2 -1

models/repo.go

reviewed

··· 25 25 "strings" 26 26 "time" 27 27 28 28 + "code.gitea.io/gitea/modules/lfs" 28 29 "code.gitea.io/gitea/modules/log" 29 30 "code.gitea.io/gitea/modules/markup" 30 31 "code.gitea.io/gitea/modules/options" ··· 1531 1532 } 1532 1533 1533 1534 for _, v := range lfsObjects { 1534 1534 - count, err := sess.Count(&LFSMetaObject{Oid: v.Oid}) 1535 1535 + count, err := sess.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: v.Oid}}) 1535 1536 if err != nil { 1536 1537 return err 1537 1538 }

+3

models/repo_mirror.go

reviewed

··· 25 25 UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` 26 26 NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` 27 27 28 28 + LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"` 29 29 + LFSEndpoint string `xorm:"lfs_endpoint TEXT"` 30 30 + 28 31 Address string `xorm:"-"` 29 32 } 30 33

+24

modules/lfs/client.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "context" 9 9 + "io" 10 10 + "net/url" 11 11 + ) 12 12 + 13 13 + // Client is used to communicate with a LFS source 14 14 + type Client interface { 15 15 + Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) 16 16 + } 17 17 + 18 18 + // NewClient creates a LFS client 19 19 + func NewClient(endpoint *url.URL) Client { 20 20 + if endpoint.Scheme == "file" { 21 21 + return newFilesystemClient(endpoint) 22 22 + } 23 23 + return newHTTPClient(endpoint) 24 24 + }

+23

modules/lfs/client_test.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "net/url" 9 9 + 10 10 + "testing" 11 11 + 12 12 + "github.com/stretchr/testify/assert" 13 13 + ) 14 14 + 15 15 + func TestNewClient(t *testing.T) { 16 16 + u, _ := url.Parse("file:///test") 17 17 + c := NewClient(u) 18 18 + assert.IsType(t, &FilesystemClient{}, c) 19 19 + 20 20 + u, _ = url.Parse("https://test.com/lfs") 21 21 + c = NewClient(u) 22 22 + assert.IsType(t, &HTTPClient{}, c) 23 23 + }

+39 -26

modules/lfs/content_store.go

reviewed

··· 13 13 "io" 14 14 "os" 15 15 16 16 - "code.gitea.io/gitea/models" 17 16 "code.gitea.io/gitea/modules/log" 18 17 "code.gitea.io/gitea/modules/storage" 19 18 ) 20 19 21 20 var ( 22 22 - errHashMismatch = errors.New("Content hash does not match OID") 23 23 - errSizeMismatch = errors.New("Content size does not match") 21 21 + // ErrHashMismatch occurs if the content has does not match OID 22 22 + ErrHashMismatch = errors.New("Content hash does not match OID") 23 23 + // ErrSizeMismatch occurs if the content size does not match 24 24 + ErrSizeMismatch = errors.New("Content size does not match") 24 25 ) 25 26 26 27 // ErrRangeNotSatisfiable represents an error which request range is not satisfiable. ··· 28 29 FromByte int64 29 30 } 30 31 31 31 - func (err ErrRangeNotSatisfiable) Error() string { 32 32 - return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte) 33 33 - } 34 34 - 35 32 // IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable 36 33 func IsErrRangeNotSatisfiable(err error) bool { 37 34 _, ok := err.(ErrRangeNotSatisfiable) 38 35 return ok 39 36 } 40 37 38 38 + func (err ErrRangeNotSatisfiable) Error() string { 39 39 + return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte) 40 40 + } 41 41 + 41 42 // ContentStore provides a simple file system based storage. 42 43 type ContentStore struct { 43 44 storage.ObjectStorage 44 45 } 45 46 47 47 + // NewContentStore creates the default ContentStore 48 48 + func NewContentStore() *ContentStore { 49 49 + contentStore := &ContentStore{ObjectStorage: storage.LFS} 50 50 + return contentStore 51 51 + } 52 52 + 46 53 // Get takes a Meta object and retrieves the content from the store, returning 47 54 // it as an io.ReadSeekCloser. 48 48 - func (s *ContentStore) Get(meta *models.LFSMetaObject) (storage.Object, error) { 49 49 - f, err := s.Open(meta.RelativePath()) 55 55 + func (s *ContentStore) Get(pointer Pointer) (storage.Object, error) { 56 56 + f, err := s.Open(pointer.RelativePath()) 50 57 if err != nil { 51 51 - log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err) 58 58 + log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", pointer.Oid, err) 52 59 return nil, err 53 60 } 54 61 return f, err 55 62 } 56 63 57 64 // Put takes a Meta object and an io.Reader and writes the content to the store. 58 58 - func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { 59 59 - p := meta.RelativePath() 65 65 + func (s *ContentStore) Put(pointer Pointer, r io.Reader) error { 66 66 + p := pointer.RelativePath() 60 67 61 68 // Wrap the provided reader with an inline hashing and size checker 62 62 - wrappedRd := newHashingReader(meta.Size, meta.Oid, r) 69 69 + wrappedRd := newHashingReader(pointer.Size, pointer.Oid, r) 63 70 64 71 // now pass the wrapped reader to Save - if there is a size mismatch or hash mismatch then 65 72 // the errors returned by the newHashingReader should percolate up to here 66 66 - written, err := s.Save(p, wrappedRd, meta.Size) 73 73 + written, err := s.Save(p, wrappedRd, pointer.Size) 67 74 if err != nil { 68 68 - log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err) 75 75 + log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", pointer.Oid, p, err) 69 76 return err 70 77 } 71 78 72 79 // This shouldn't happen but it is sensible to test 73 73 - if written != meta.Size { 80 80 + if written != pointer.Size { 74 81 if err := s.Delete(p); err != nil { 75 75 - log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err) 82 82 + log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err) 76 83 } 77 77 - return errSizeMismatch 84 84 + return ErrSizeMismatch 78 85 } 79 86 80 87 return nil 81 88 } 82 89 83 90 // Exists returns true if the object exists in the content store. 84 84 - func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) { 85 85 - _, err := s.ObjectStorage.Stat(meta.RelativePath()) 91 91 + func (s *ContentStore) Exists(pointer Pointer) (bool, error) { 92 92 + _, err := s.ObjectStorage.Stat(pointer.RelativePath()) 86 93 if err != nil { 87 94 if os.IsNotExist(err) { 88 95 return false, nil ··· 93 100 } 94 101 95 102 // Verify returns true if the object exists in the content store and size is correct. 96 96 - func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) { 97 97 - p := meta.RelativePath() 103 103 + func (s *ContentStore) Verify(pointer Pointer) (bool, error) { 104 104 + p := pointer.RelativePath() 98 105 fi, err := s.ObjectStorage.Stat(p) 99 99 - if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) { 106 106 + if os.IsNotExist(err) || (err == nil && fi.Size() != pointer.Size) { 100 107 return false, nil 101 108 } else if err != nil { 102 102 - log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err) 109 109 + log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, pointer.Oid, err) 103 110 return false, err 104 111 } 105 112 106 113 return true, nil 114 114 + } 115 115 + 116 116 + // ReadMetaObject will read a models.LFSMetaObject and return a reader 117 117 + func ReadMetaObject(pointer Pointer) (io.ReadCloser, error) { 118 118 + contentStore := NewContentStore() 119 119 + return contentStore.Get(pointer) 107 120 } 108 121 109 122 type hashingReader struct { ··· 127 140 128 141 if err != nil && err == io.EOF { 129 142 if r.currentSize != r.expectedSize { 130 130 - return n, errSizeMismatch 143 143 + return n, ErrSizeMismatch 131 144 } 132 145 133 146 shaStr := hex.EncodeToString(r.hash.Sum(nil)) 134 147 if shaStr != r.expectedHash { 135 135 - return n, errHashMismatch 148 148 + return n, ErrHashMismatch 136 149 } 137 150 } 138 151

+106

modules/lfs/endpoint.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "fmt" 9 9 + "net/url" 10 10 + "os" 11 11 + "path" 12 12 + "path/filepath" 13 13 + "strings" 14 14 + 15 15 + "code.gitea.io/gitea/modules/log" 16 16 + ) 17 17 + 18 18 + // DetermineEndpoint determines an endpoint from the clone url or uses the specified LFS url. 19 19 + func DetermineEndpoint(cloneurl, lfsurl string) *url.URL { 20 20 + if len(lfsurl) > 0 { 21 21 + return endpointFromURL(lfsurl) 22 22 + } 23 23 + return endpointFromCloneURL(cloneurl) 24 24 + } 25 25 + 26 26 + func endpointFromCloneURL(rawurl string) *url.URL { 27 27 + ep := endpointFromURL(rawurl) 28 28 + if ep == nil { 29 29 + return ep 30 30 + } 31 31 + 32 32 + if strings.HasSuffix(ep.Path, "/") { 33 33 + ep.Path = ep.Path[:len(ep.Path)-1] 34 34 + } 35 35 + 36 36 + if ep.Scheme == "file" { 37 37 + return ep 38 38 + } 39 39 + 40 40 + if path.Ext(ep.Path) == ".git" { 41 41 + ep.Path += "/info/lfs" 42 42 + } else { 43 43 + ep.Path += ".git/info/lfs" 44 44 + } 45 45 + 46 46 + return ep 47 47 + } 48 48 + 49 49 + func endpointFromURL(rawurl string) *url.URL { 50 50 + if strings.HasPrefix(rawurl, "/") { 51 51 + return endpointFromLocalPath(rawurl) 52 52 + } 53 53 + 54 54 + u, err := url.Parse(rawurl) 55 55 + if err != nil { 56 56 + log.Error("lfs.endpointFromUrl: %v", err) 57 57 + return nil 58 58 + } 59 59 + 60 60 + switch u.Scheme { 61 61 + case "http", "https": 62 62 + return u 63 63 + case "git": 64 64 + u.Scheme = "https" 65 65 + return u 66 66 + case "file": 67 67 + return u 68 68 + default: 69 69 + if _, err := os.Stat(rawurl); err == nil { 70 70 + return endpointFromLocalPath(rawurl) 71 71 + } 72 72 + 73 73 + log.Error("lfs.endpointFromUrl: unknown url") 74 74 + return nil 75 75 + } 76 76 + } 77 77 + 78 78 + func endpointFromLocalPath(path string) *url.URL { 79 79 + var slash string 80 80 + if abs, err := filepath.Abs(path); err == nil { 81 81 + if !strings.HasPrefix(abs, "/") { 82 82 + slash = "/" 83 83 + } 84 84 + path = abs 85 85 + } 86 86 + 87 87 + var gitpath string 88 88 + if filepath.Base(path) == ".git" { 89 89 + gitpath = path 90 90 + path = filepath.Dir(path) 91 91 + } else { 92 92 + gitpath = filepath.Join(path, ".git") 93 93 + } 94 94 + 95 95 + if _, err := os.Stat(gitpath); err == nil { 96 96 + path = gitpath 97 97 + } else if _, err := os.Stat(path); err != nil { 98 98 + return nil 99 99 + } 100 100 + 101 101 + path = fmt.Sprintf("file://%s%s", slash, filepath.ToSlash(path)) 102 102 + 103 103 + u, _ := url.Parse(path) 104 104 + 105 105 + return u 106 106 + }

+75

modules/lfs/endpoint_test.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "net/url" 9 9 + "testing" 10 10 + 11 11 + "github.com/stretchr/testify/assert" 12 12 + ) 13 13 + 14 14 + func str2url(raw string) *url.URL { 15 15 + u, _ := url.Parse(raw) 16 16 + return u 17 17 + } 18 18 + 19 19 + func TestDetermineEndpoint(t *testing.T) { 20 20 + // Test cases 21 21 + var cases = []struct { 22 22 + cloneurl string 23 23 + lfsurl string 24 24 + expected *url.URL 25 25 + }{ 26 26 + // case 0 27 27 + { 28 28 + cloneurl: "", 29 29 + lfsurl: "", 30 30 + expected: nil, 31 31 + }, 32 32 + // case 1 33 33 + { 34 34 + cloneurl: "https://git.com/repo", 35 35 + lfsurl: "", 36 36 + expected: str2url("https://git.com/repo.git/info/lfs"), 37 37 + }, 38 38 + // case 2 39 39 + { 40 40 + cloneurl: "https://git.com/repo.git", 41 41 + lfsurl: "", 42 42 + expected: str2url("https://git.com/repo.git/info/lfs"), 43 43 + }, 44 44 + // case 3 45 45 + { 46 46 + cloneurl: "", 47 47 + lfsurl: "https://gitlfs.com/repo", 48 48 + expected: str2url("https://gitlfs.com/repo"), 49 49 + }, 50 50 + // case 4 51 51 + { 52 52 + cloneurl: "https://git.com/repo.git", 53 53 + lfsurl: "https://gitlfs.com/repo", 54 54 + expected: str2url("https://gitlfs.com/repo"), 55 55 + }, 56 56 + // case 5 57 57 + { 58 58 + cloneurl: "git://git.com/repo.git", 59 59 + lfsurl: "", 60 60 + expected: str2url("https://git.com/repo.git/info/lfs"), 61 61 + }, 62 62 + // case 6 63 63 + { 64 64 + cloneurl: "", 65 65 + lfsurl: "git://gitlfs.com/repo", 66 66 + expected: str2url("https://gitlfs.com/repo"), 67 67 + }, 68 68 + } 69 69 + 70 70 + for n, c := range cases { 71 71 + ep := DetermineEndpoint(c.cloneurl, c.lfsurl) 72 72 + 73 73 + assert.Equal(t, c.expected, ep, "case %d: error should match", n) 74 74 + } 75 75 + }

+50

modules/lfs/filesystem_client.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "context" 9 9 + "io" 10 10 + "net/url" 11 11 + "os" 12 12 + "path/filepath" 13 13 + 14 14 + "code.gitea.io/gitea/modules/util" 15 15 + ) 16 16 + 17 17 + // FilesystemClient is used to read LFS data from a filesystem path 18 18 + type FilesystemClient struct { 19 19 + lfsdir string 20 20 + } 21 21 + 22 22 + func newFilesystemClient(endpoint *url.URL) *FilesystemClient { 23 23 + path, _ := util.FileURLToPath(endpoint) 24 24 + 25 25 + lfsdir := filepath.Join(path, "lfs", "objects") 26 26 + 27 27 + client := &FilesystemClient{lfsdir} 28 28 + 29 29 + return client 30 30 + } 31 31 + 32 32 + func (c *FilesystemClient) objectPath(oid string) string { 33 33 + return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid) 34 34 + } 35 35 + 36 36 + // Download reads the specific LFS object from the target repository 37 37 + func (c *FilesystemClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { 38 38 + objectPath := c.objectPath(oid) 39 39 + 40 40 + if _, err := os.Stat(objectPath); os.IsNotExist(err) { 41 41 + return nil, err 42 42 + } 43 43 + 44 44 + file, err := os.Open(objectPath) 45 45 + if err != nil { 46 46 + return nil, err 47 47 + } 48 48 + 49 49 + return file, nil 50 50 + }

+129

modules/lfs/http_client.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "bytes" 9 9 + "context" 10 10 + "encoding/json" 11 11 + "errors" 12 12 + "fmt" 13 13 + "io" 14 14 + "net/http" 15 15 + "net/url" 16 16 + "strings" 17 17 + 18 18 + "code.gitea.io/gitea/modules/log" 19 19 + ) 20 20 + 21 21 + // HTTPClient is used to communicate with the LFS server 22 22 + // https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md 23 23 + type HTTPClient struct { 24 24 + client *http.Client 25 25 + endpoint string 26 26 + transfers map[string]TransferAdapter 27 27 + } 28 28 + 29 29 + func newHTTPClient(endpoint *url.URL) *HTTPClient { 30 30 + hc := &http.Client{} 31 31 + 32 32 + client := &HTTPClient{ 33 33 + client: hc, 34 34 + endpoint: strings.TrimSuffix(endpoint.String(), "/"), 35 35 + transfers: make(map[string]TransferAdapter), 36 36 + } 37 37 + 38 38 + basic := &BasicTransferAdapter{hc} 39 39 + 40 40 + client.transfers[basic.Name()] = basic 41 41 + 42 42 + return client 43 43 + } 44 44 + 45 45 + func (c *HTTPClient) transferNames() []string { 46 46 + keys := make([]string, len(c.transfers)) 47 47 + 48 48 + i := 0 49 49 + for k := range c.transfers { 50 50 + keys[i] = k 51 51 + i++ 52 52 + } 53 53 + 54 54 + return keys 55 55 + } 56 56 + 57 57 + func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { 58 58 + url := fmt.Sprintf("%s/objects/batch", c.endpoint) 59 59 + 60 60 + request := &BatchRequest{operation, c.transferNames(), nil, objects} 61 61 + 62 62 + payload := new(bytes.Buffer) 63 63 + err := json.NewEncoder(payload).Encode(request) 64 64 + if err != nil { 65 65 + return nil, fmt.Errorf("lfs.HTTPClient.batch json.Encode: %w", err) 66 66 + } 67 67 + 68 68 + log.Trace("lfs.HTTPClient.batch NewRequestWithContext: %s", url) 69 69 + 70 70 + req, err := http.NewRequestWithContext(ctx, "POST", url, payload) 71 71 + if err != nil { 72 72 + return nil, fmt.Errorf("lfs.HTTPClient.batch http.NewRequestWithContext: %w", err) 73 73 + } 74 74 + req.Header.Set("Content-type", MediaType) 75 75 + req.Header.Set("Accept", MediaType) 76 76 + 77 77 + res, err := c.client.Do(req) 78 78 + if err != nil { 79 79 + select { 80 80 + case <-ctx.Done(): 81 81 + return nil, ctx.Err() 82 82 + default: 83 83 + } 84 84 + return nil, fmt.Errorf("lfs.HTTPClient.batch http.Do: %w", err) 85 85 + } 86 86 + defer res.Body.Close() 87 87 + 88 88 + if res.StatusCode != http.StatusOK { 89 89 + return nil, fmt.Errorf("lfs.HTTPClient.batch: Unexpected servers response: %s", res.Status) 90 90 + } 91 91 + 92 92 + var response BatchResponse 93 93 + err = json.NewDecoder(res.Body).Decode(&response) 94 94 + if err != nil { 95 95 + return nil, fmt.Errorf("lfs.HTTPClient.batch json.Decode: %w", err) 96 96 + } 97 97 + 98 98 + if len(response.Transfer) == 0 { 99 99 + response.Transfer = "basic" 100 100 + } 101 101 + 102 102 + return &response, nil 103 103 + } 104 104 + 105 105 + // Download reads the specific LFS object from the LFS server 106 106 + func (c *HTTPClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { 107 107 + var objects []Pointer 108 108 + objects = append(objects, Pointer{oid, size}) 109 109 + 110 110 + result, err := c.batch(ctx, "download", objects) 111 111 + if err != nil { 112 112 + return nil, err 113 113 + } 114 114 + 115 115 + transferAdapter, ok := c.transfers[result.Transfer] 116 116 + if !ok { 117 117 + return nil, fmt.Errorf("lfs.HTTPClient.Download Transferadapter not found: %s", result.Transfer) 118 118 + } 119 119 + 120 120 + if len(result.Objects) == 0 { 121 121 + return nil, errors.New("lfs.HTTPClient.Download: No objects in result") 122 122 + } 123 123 + 124 124 + content, err := transferAdapter.Download(ctx, result.Objects[0]) 125 125 + if err != nil { 126 126 + return nil, err 127 127 + } 128 128 + return content, nil 129 129 + }

+144

modules/lfs/http_client_test.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "bytes" 9 9 + "context" 10 10 + "encoding/json" 11 11 + "io" 12 12 + "io/ioutil" 13 13 + "net/http" 14 14 + "strings" 15 15 + "testing" 16 16 + 17 17 + "github.com/stretchr/testify/assert" 18 18 + ) 19 19 + 20 20 + type RoundTripFunc func(req *http.Request) *http.Response 21 21 + 22 22 + func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { 23 23 + return f(req), nil 24 24 + } 25 25 + 26 26 + type DummyTransferAdapter struct { 27 27 + } 28 28 + 29 29 + func (a *DummyTransferAdapter) Name() string { 30 30 + return "dummy" 31 31 + } 32 32 + 33 33 + func (a *DummyTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { 34 34 + return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil 35 35 + } 36 36 + 37 37 + func TestHTTPClientDownload(t *testing.T) { 38 38 + oid := "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041" 39 39 + size := int64(6) 40 40 + 41 41 + roundTripHandler := func(req *http.Request) *http.Response { 42 42 + url := req.URL.String() 43 43 + if strings.Contains(url, "status-not-ok") { 44 44 + return &http.Response{StatusCode: http.StatusBadRequest} 45 45 + } 46 46 + if strings.Contains(url, "invalid-json-response") { 47 47 + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))} 48 48 + } 49 49 + if strings.Contains(url, "valid-batch-request-download") { 50 50 + assert.Equal(t, "POST", req.Method) 51 51 + assert.Equal(t, MediaType, req.Header.Get("Content-type"), "case %s: error should match", url) 52 52 + assert.Equal(t, MediaType, req.Header.Get("Accept"), "case %s: error should match", url) 53 53 + 54 54 + var batchRequest BatchRequest 55 55 + err := json.NewDecoder(req.Body).Decode(&batchRequest) 56 56 + assert.NoError(t, err) 57 57 + 58 58 + assert.Equal(t, "download", batchRequest.Operation) 59 59 + assert.Equal(t, 1, len(batchRequest.Objects)) 60 60 + assert.Equal(t, oid, batchRequest.Objects[0].Oid) 61 61 + assert.Equal(t, size, batchRequest.Objects[0].Size) 62 62 + 63 63 + batchResponse := &BatchResponse{ 64 64 + Transfer: "dummy", 65 65 + Objects: make([]*ObjectResponse, 1), 66 66 + } 67 67 + 68 68 + payload := new(bytes.Buffer) 69 69 + json.NewEncoder(payload).Encode(batchResponse) 70 70 + 71 71 + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} 72 72 + } 73 73 + if strings.Contains(url, "invalid-response-no-objects") { 74 74 + batchResponse := &BatchResponse{Transfer: "dummy"} 75 75 + 76 76 + payload := new(bytes.Buffer) 77 77 + json.NewEncoder(payload).Encode(batchResponse) 78 78 + 79 79 + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} 80 80 + } 81 81 + if strings.Contains(url, "unknown-transfer-adapter") { 82 82 + batchResponse := &BatchResponse{Transfer: "unknown_adapter"} 83 83 + 84 84 + payload := new(bytes.Buffer) 85 85 + json.NewEncoder(payload).Encode(batchResponse) 86 86 + 87 87 + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} 88 88 + } 89 89 + 90 90 + t.Errorf("Unknown test case: %s", url) 91 91 + 92 92 + return nil 93 93 + } 94 94 + 95 95 + hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)} 96 96 + dummy := &DummyTransferAdapter{} 97 97 + 98 98 + var cases = []struct { 99 99 + endpoint string 100 100 + expectederror string 101 101 + }{ 102 102 + // case 0 103 103 + { 104 104 + endpoint: "https://status-not-ok.io", 105 105 + expectederror: "Unexpected servers response: ", 106 106 + }, 107 107 + // case 1 108 108 + { 109 109 + endpoint: "https://invalid-json-response.io", 110 110 + expectederror: "json.Decode: ", 111 111 + }, 112 112 + // case 2 113 113 + { 114 114 + endpoint: "https://valid-batch-request-download.io", 115 115 + expectederror: "", 116 116 + }, 117 117 + // case 3 118 118 + { 119 119 + endpoint: "https://invalid-response-no-objects.io", 120 120 + expectederror: "No objects in result", 121 121 + }, 122 122 + // case 4 123 123 + { 124 124 + endpoint: "https://unknown-transfer-adapter.io", 125 125 + expectederror: "Transferadapter not found: ", 126 126 + }, 127 127 + } 128 128 + 129 129 + for n, c := range cases { 130 130 + client := &HTTPClient{ 131 131 + client: hc, 132 132 + endpoint: c.endpoint, 133 133 + transfers: make(map[string]TransferAdapter), 134 134 + } 135 135 + client.transfers["dummy"] = dummy 136 136 + 137 137 + _, err := client.Download(context.Background(), oid, size) 138 138 + if len(c.expectederror) > 0 { 139 139 + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) 140 140 + } else { 141 141 + assert.NoError(t, err, "case %d", n) 142 142 + } 143 143 + } 144 144 + }

+7 -6

modules/lfs/locks.go services/lfs/locks.go

reviewed

··· 12 12 "code.gitea.io/gitea/models" 13 13 "code.gitea.io/gitea/modules/context" 14 14 "code.gitea.io/gitea/modules/convert" 15 15 + lfs_module "code.gitea.io/gitea/modules/lfs" 15 16 "code.gitea.io/gitea/modules/log" 16 17 "code.gitea.io/gitea/modules/setting" 17 18 api "code.gitea.io/gitea/modules/structs" ··· 26 27 return false 27 28 } 28 29 if !MetaMatcher(ctx.Req) { 29 29 - log.Info("Attempt access LOCKs without accepting the correct media type: %s", metaMediaType) 30 30 + log.Info("Attempt access LOCKs without accepting the correct media type: %s", lfs_module.MediaType) 30 31 writeStatus(ctx, http.StatusBadRequest) 31 32 return false 32 33 } ··· 72 73 // Status is written in checkIsValidRequest 73 74 return 74 75 } 75 75 - ctx.Resp.Header().Set("Content-Type", metaMediaType) 76 76 + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) 76 77 77 77 - rv := unpack(ctx) 78 78 + rv, _ := unpack(ctx) 78 79 79 80 repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo) 80 81 if err != nil { ··· 159 160 // Status is written in checkIsValidRequest 160 161 return 161 162 } 162 162 - ctx.Resp.Header().Set("Content-Type", metaMediaType) 163 163 + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) 163 164 164 165 userName := ctx.Params("username") 165 166 repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git") ··· 228 229 // Status is written in checkIsValidRequest 229 230 return 230 231 } 231 231 - ctx.Resp.Header().Set("Content-Type", metaMediaType) 232 232 + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) 232 233 233 234 userName := ctx.Params("username") 234 235 repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git") ··· 295 296 // Status is written in checkIsValidRequest 296 297 return 297 298 } 298 298 - ctx.Resp.Header().Set("Content-Type", metaMediaType) 299 299 + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) 299 300 300 301 userName := ctx.Params("username") 301 302 repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")

+123

modules/lfs/pointer.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "crypto/sha256" 9 9 + "encoding/hex" 10 10 + "errors" 11 11 + "fmt" 12 12 + "io" 13 13 + "path" 14 14 + "regexp" 15 15 + "strconv" 16 16 + "strings" 17 17 + ) 18 18 + 19 19 + const ( 20 20 + blobSizeCutoff = 1024 21 21 + 22 22 + // MetaFileIdentifier is the string appearing at the first line of LFS pointer files. 23 23 + // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md 24 24 + MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" 25 25 + 26 26 + // MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. 27 27 + MetaFileOidPrefix = "oid sha256:" 28 28 + ) 29 29 + 30 30 + var ( 31 31 + // ErrMissingPrefix occurs if the content lacks the LFS prefix 32 32 + ErrMissingPrefix = errors.New("Content lacks the LFS prefix") 33 33 + 34 34 + // ErrInvalidStructure occurs if the content has an invalid structure 35 35 + ErrInvalidStructure = errors.New("Content has an invalid structure") 36 36 + 37 37 + // ErrInvalidOIDFormat occurs if the oid has an invalid format 38 38 + ErrInvalidOIDFormat = errors.New("OID has an invalid format") 39 39 + ) 40 40 + 41 41 + // ReadPointer tries to read LFS pointer data from the reader 42 42 + func ReadPointer(reader io.Reader) (Pointer, error) { 43 43 + buf := make([]byte, blobSizeCutoff) 44 44 + n, err := io.ReadFull(reader, buf) 45 45 + if err != nil && err != io.ErrUnexpectedEOF { 46 46 + return Pointer{}, err 47 47 + } 48 48 + buf = buf[:n] 49 49 + 50 50 + return ReadPointerFromBuffer(buf) 51 51 + } 52 52 + 53 53 + var oidPattern = regexp.MustCompile(`^[a-f\d]{64}$`) 54 54 + 55 55 + // ReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or an error otherwise. 56 56 + func ReadPointerFromBuffer(buf []byte) (Pointer, error) { 57 57 + var p Pointer 58 58 + 59 59 + headString := string(buf) 60 60 + if !strings.HasPrefix(headString, MetaFileIdentifier) { 61 61 + return p, ErrMissingPrefix 62 62 + } 63 63 + 64 64 + splitLines := strings.Split(headString, "\n") 65 65 + if len(splitLines) < 3 { 66 66 + return p, ErrInvalidStructure 67 67 + } 68 68 + 69 69 + oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix) 70 70 + if len(oid) != 64 || !oidPattern.MatchString(oid) { 71 71 + return p, ErrInvalidOIDFormat 72 72 + } 73 73 + size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) 74 74 + if err != nil { 75 75 + return p, err 76 76 + } 77 77 + 78 78 + p.Oid = oid 79 79 + p.Size = size 80 80 + 81 81 + return p, nil 82 82 + } 83 83 + 84 84 + // IsValid checks if the pointer has a valid structure. 85 85 + // It doesn't check if the pointed-to-content exists. 86 86 + func (p Pointer) IsValid() bool { 87 87 + if len(p.Oid) != 64 { 88 88 + return false 89 89 + } 90 90 + if !oidPattern.MatchString(p.Oid) { 91 91 + return false 92 92 + } 93 93 + if p.Size < 0 { 94 94 + return false 95 95 + } 96 96 + return true 97 97 + } 98 98 + 99 99 + // StringContent returns the string representation of the pointer 100 100 + // https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#the-pointer 101 101 + func (p Pointer) StringContent() string { 102 102 + return fmt.Sprintf("%s\n%s%s\nsize %d\n", MetaFileIdentifier, MetaFileOidPrefix, p.Oid, p.Size) 103 103 + } 104 104 + 105 105 + // RelativePath returns the relative storage path of the pointer 106 106 + func (p Pointer) RelativePath() string { 107 107 + if len(p.Oid) < 5 { 108 108 + return p.Oid 109 109 + } 110 110 + 111 111 + return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:]) 112 112 + } 113 113 + 114 114 + // GeneratePointer generates a pointer for arbitrary content 115 115 + func GeneratePointer(content io.Reader) (Pointer, error) { 116 116 + h := sha256.New() 117 117 + c, err := io.Copy(h, content) 118 118 + if err != nil { 119 119 + return Pointer{}, err 120 120 + } 121 121 + sum := h.Sum(nil) 122 122 + return Pointer{Oid: hex.EncodeToString(sum), Size: c}, nil 123 123 + }

+64

modules/lfs/pointer_scanner_gogit.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + // +build gogit 6 6 + 7 7 + package lfs 8 8 + 9 9 + import ( 10 10 + "context" 11 11 + "fmt" 12 12 + 13 13 + "code.gitea.io/gitea/modules/git" 14 14 + 15 15 + "github.com/go-git/go-git/v5/plumbing/object" 16 16 + ) 17 17 + 18 18 + // SearchPointerBlobs scans the whole repository for LFS pointer files 19 19 + func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) { 20 20 + gitRepo := repo.GoGitRepo() 21 21 + 22 22 + err := func() error { 23 23 + blobs, err := gitRepo.BlobObjects() 24 24 + if err != nil { 25 25 + return fmt.Errorf("lfs.SearchPointerBlobs BlobObjects: %w", err) 26 26 + } 27 27 + 28 28 + return blobs.ForEach(func(blob *object.Blob) error { 29 29 + select { 30 30 + case <-ctx.Done(): 31 31 + return ctx.Err() 32 32 + default: 33 33 + } 34 34 + 35 35 + if blob.Size > blobSizeCutoff { 36 36 + return nil 37 37 + } 38 38 + 39 39 + reader, err := blob.Reader() 40 40 + if err != nil { 41 41 + return fmt.Errorf("lfs.SearchPointerBlobs blob.Reader: %w", err) 42 42 + } 43 43 + defer reader.Close() 44 44 + 45 45 + pointer, _ := ReadPointer(reader) 46 46 + if pointer.IsValid() { 47 47 + pointerChan <- PointerBlob{Hash: blob.Hash.String(), Pointer: pointer} 48 48 + } 49 49 + 50 50 + return nil 51 51 + }) 52 52 + }() 53 53 + 54 54 + if err != nil { 55 55 + select { 56 56 + case <-ctx.Done(): 57 57 + default: 58 58 + errChan <- err 59 59 + } 60 60 + } 61 61 + 62 62 + close(pointerChan) 63 63 + close(errChan) 64 64 + }

+110

modules/lfs/pointer_scanner_nogogit.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + // +build !gogit 6 6 + 7 7 + package lfs 8 8 + 9 9 + import ( 10 10 + "bufio" 11 11 + "context" 12 12 + "io" 13 13 + "strconv" 14 14 + "sync" 15 15 + 16 16 + "code.gitea.io/gitea/modules/git" 17 17 + "code.gitea.io/gitea/modules/git/pipeline" 18 18 + ) 19 19 + 20 20 + // SearchPointerBlobs scans the whole repository for LFS pointer files 21 21 + func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) { 22 22 + basePath := repo.Path 23 23 + 24 24 + catFileCheckReader, catFileCheckWriter := io.Pipe() 25 25 + shasToBatchReader, shasToBatchWriter := io.Pipe() 26 26 + catFileBatchReader, catFileBatchWriter := io.Pipe() 27 27 + 28 28 + wg := sync.WaitGroup{} 29 29 + wg.Add(4) 30 30 + 31 31 + // Create the go-routines in reverse order. 32 32 + 33 33 + // 4. Take the output of cat-file --batch and check if each file in turn 34 34 + // to see if they're pointers to files in the LFS store 35 35 + go createPointerResultsFromCatFileBatch(ctx, catFileBatchReader, &wg, pointerChan) 36 36 + 37 37 + // 3. Take the shas of the blobs and batch read them 38 38 + go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) 39 39 + 40 40 + // 2. From the provided objects restrict to blobs <=1k 41 41 + go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) 42 42 + 43 43 + // 1. Run batch-check on all objects in the repository 44 44 + if git.CheckGitVersionAtLeast("2.6.0") != nil { 45 45 + revListReader, revListWriter := io.Pipe() 46 46 + shasToCheckReader, shasToCheckWriter := io.Pipe() 47 47 + wg.Add(2) 48 48 + go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath) 49 49 + go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg) 50 50 + go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan) 51 51 + } else { 52 52 + go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan) 53 53 + } 54 54 + wg.Wait() 55 55 + 56 56 + close(pointerChan) 57 57 + close(errChan) 58 58 + } 59 59 + 60 60 + func createPointerResultsFromCatFileBatch(ctx context.Context, catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- PointerBlob) { 61 61 + defer wg.Done() 62 62 + defer catFileBatchReader.Close() 63 63 + 64 64 + bufferedReader := bufio.NewReader(catFileBatchReader) 65 65 + buf := make([]byte, 1025) 66 66 + 67 67 + loop: 68 68 + for { 69 69 + select { 70 70 + case <-ctx.Done(): 71 71 + break loop 72 72 + default: 73 73 + } 74 74 + 75 75 + // File descriptor line: sha 76 76 + sha, err := bufferedReader.ReadString(' ') 77 77 + if err != nil { 78 78 + _ = catFileBatchReader.CloseWithError(err) 79 79 + break 80 80 + } 81 81 + // Throw away the blob 82 82 + if _, err := bufferedReader.ReadString(' '); err != nil { 83 83 + _ = catFileBatchReader.CloseWithError(err) 84 84 + break 85 85 + } 86 86 + sizeStr, err := bufferedReader.ReadString('\n') 87 87 + if err != nil { 88 88 + _ = catFileBatchReader.CloseWithError(err) 89 89 + break 90 90 + } 91 91 + size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1]) 92 92 + if err != nil { 93 93 + _ = catFileBatchReader.CloseWithError(err) 94 94 + break 95 95 + } 96 96 + pointerBuf := buf[:size+1] 97 97 + if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil { 98 98 + _ = catFileBatchReader.CloseWithError(err) 99 99 + break 100 100 + } 101 101 + pointerBuf = pointerBuf[:size] 102 102 + // Now we need to check if the pointerBuf is an LFS pointer 103 103 + pointer, _ := ReadPointerFromBuffer(pointerBuf) 104 104 + if !pointer.IsValid() { 105 105 + continue 106 106 + } 107 107 + 108 108 + pointerChan <- PointerBlob{Hash: sha, Pointer: pointer} 109 109 + } 110 110 + }

+103

modules/lfs/pointer_test.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "path" 9 9 + "strings" 10 10 + "testing" 11 11 + 12 12 + "github.com/stretchr/testify/assert" 13 13 + ) 14 14 + 15 15 + func TestStringContent(t *testing.T) { 16 16 + p := Pointer{Oid: "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393", Size: 1234} 17 17 + expected := "version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n" 18 18 + assert.Equal(t, p.StringContent(), expected) 19 19 + } 20 20 + 21 21 + func TestRelativePath(t *testing.T) { 22 22 + p := Pointer{Oid: "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393"} 23 23 + expected := path.Join("4d", "7a", "214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") 24 24 + assert.Equal(t, p.RelativePath(), expected) 25 25 + 26 26 + p2 := Pointer{Oid: "4d7a"} 27 27 + assert.Equal(t, p2.RelativePath(), "4d7a") 28 28 + } 29 29 + 30 30 + func TestIsValid(t *testing.T) { 31 31 + p := Pointer{} 32 32 + assert.False(t, p.IsValid()) 33 33 + 34 34 + p = Pointer{Oid: "123"} 35 35 + assert.False(t, p.IsValid()) 36 36 + 37 37 + p = Pointer{Oid: "z4cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc"} 38 38 + assert.False(t, p.IsValid()) 39 39 + 40 40 + p = Pointer{Oid: "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc"} 41 41 + assert.True(t, p.IsValid()) 42 42 + 43 43 + p = Pointer{Oid: "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc", Size: -1} 44 44 + assert.False(t, p.IsValid()) 45 45 + } 46 46 + 47 47 + func TestGeneratePointer(t *testing.T) { 48 48 + p, err := GeneratePointer(strings.NewReader("Gitea")) 49 49 + assert.NoError(t, err) 50 50 + assert.True(t, p.IsValid()) 51 51 + assert.Equal(t, p.Oid, "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc") 52 52 + assert.Equal(t, p.Size, int64(5)) 53 53 + } 54 54 + 55 55 + func TestReadPointerFromBuffer(t *testing.T) { 56 56 + p, err := ReadPointerFromBuffer([]byte{}) 57 57 + assert.ErrorIs(t, err, ErrMissingPrefix) 58 58 + assert.False(t, p.IsValid()) 59 59 + 60 60 + p, err = ReadPointerFromBuffer([]byte("test")) 61 61 + assert.ErrorIs(t, err, ErrMissingPrefix) 62 62 + assert.False(t, p.IsValid()) 63 63 + 64 64 + p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\n")) 65 65 + assert.ErrorIs(t, err, ErrInvalidStructure) 66 66 + assert.False(t, p.IsValid()) 67 67 + 68 68 + p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a\nsize 1234\n")) 69 69 + assert.ErrorIs(t, err, ErrInvalidOIDFormat) 70 70 + assert.False(t, p.IsValid()) 71 71 + 72 72 + p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a2146z4ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n")) 73 73 + assert.ErrorIs(t, err, ErrInvalidOIDFormat) 74 74 + assert.False(t, p.IsValid()) 75 75 + 76 76 + p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\ntest 1234\n")) 77 77 + assert.Error(t, err) 78 78 + assert.False(t, p.IsValid()) 79 79 + 80 80 + p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize test\n")) 81 81 + assert.Error(t, err) 82 82 + assert.False(t, p.IsValid()) 83 83 + 84 84 + p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n")) 85 85 + assert.NoError(t, err) 86 86 + assert.True(t, p.IsValid()) 87 87 + assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") 88 88 + assert.Equal(t, p.Size, int64(1234)) 89 89 + 90 90 + p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\ntest")) 91 91 + assert.NoError(t, err) 92 92 + assert.True(t, p.IsValid()) 93 93 + assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") 94 94 + assert.Equal(t, p.Size, int64(1234)) 95 95 + } 96 96 + 97 97 + func TestReadPointer(t *testing.T) { 98 98 + p, err := ReadPointer(strings.NewReader("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n")) 99 99 + assert.NoError(t, err) 100 100 + assert.True(t, p.IsValid()) 101 101 + assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") 102 102 + assert.Equal(t, p.Size, int64(1234)) 103 103 + }

-71

modules/lfs/pointers.go

reviewed

··· 1 1 - // Copyright 2019 The Gitea Authors. All rights reserved. 2 2 - // Use of this source code is governed by a MIT-style 3 3 - // license that can be found in the LICENSE file. 4 4 - 5 5 - package lfs 6 6 - 7 7 - import ( 8 8 - "io" 9 9 - "strconv" 10 10 - "strings" 11 11 - 12 12 - "code.gitea.io/gitea/models" 13 13 - "code.gitea.io/gitea/modules/base" 14 14 - "code.gitea.io/gitea/modules/setting" 15 15 - "code.gitea.io/gitea/modules/storage" 16 16 - ) 17 17 - 18 18 - // ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file 19 19 - func ReadPointerFile(reader io.Reader) (*models.LFSMetaObject, *[]byte) { 20 20 - if !setting.LFS.StartServer { 21 21 - return nil, nil 22 22 - } 23 23 - 24 24 - buf := make([]byte, 1024) 25 25 - n, _ := reader.Read(buf) 26 26 - buf = buf[:n] 27 27 - 28 28 - if isTextFile := base.IsTextFile(buf); !isTextFile { 29 29 - return nil, nil 30 30 - } 31 31 - 32 32 - return IsPointerFile(&buf), &buf 33 33 - } 34 34 - 35 35 - // IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file 36 36 - func IsPointerFile(buf *[]byte) *models.LFSMetaObject { 37 37 - if !setting.LFS.StartServer { 38 38 - return nil 39 39 - } 40 40 - 41 41 - headString := string(*buf) 42 42 - if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) { 43 43 - return nil 44 44 - } 45 45 - 46 46 - splitLines := strings.Split(headString, "\n") 47 47 - if len(splitLines) < 3 { 48 48 - return nil 49 49 - } 50 50 - 51 51 - oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix) 52 52 - size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) 53 53 - if len(oid) != 64 || err != nil { 54 54 - return nil 55 55 - } 56 56 - 57 57 - contentStore := &ContentStore{ObjectStorage: storage.LFS} 58 58 - meta := &models.LFSMetaObject{Oid: oid, Size: size} 59 59 - exist, err := contentStore.Exists(meta) 60 60 - if err != nil || !exist { 61 61 - return nil 62 62 - } 63 63 - 64 64 - return meta 65 65 - } 66 66 - 67 67 - // ReadMetaObject will read a models.LFSMetaObject and return a reader 68 68 - func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) { 69 69 - contentStore := &ContentStore{ObjectStorage: storage.LFS} 70 70 - return contentStore.Get(meta) 71 71 - }

+100 -146

modules/lfs/server.go services/lfs/server.go

reviewed

··· 13 13 "regexp" 14 14 "strconv" 15 15 "strings" 16 16 - "time" 17 16 18 17 "code.gitea.io/gitea/models" 19 18 "code.gitea.io/gitea/modules/context" 19 19 + lfs_module "code.gitea.io/gitea/modules/lfs" 20 20 "code.gitea.io/gitea/modules/log" 21 21 "code.gitea.io/gitea/modules/setting" 22 22 - "code.gitea.io/gitea/modules/storage" 23 22 24 23 "github.com/dgrijalva/jwt-go" 25 24 jsoniter "github.com/json-iterator/go" 26 25 ) 27 26 28 28 - const ( 29 29 - metaMediaType = "application/vnd.git-lfs+json" 30 30 - ) 31 31 - 32 32 - // RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and 33 33 - // some headers are stored. 34 34 - type RequestVars struct { 35 35 - Oid string 36 36 - Size int64 27 27 + // requestContext contain variables from the HTTP request. 28 28 + type requestContext struct { 37 29 User string 38 38 - Password string 39 30 Repo string 40 31 Authorization string 41 32 } 42 33 43 43 - // BatchVars contains multiple RequestVars processed in one batch operation. 44 44 - // https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md 45 45 - type BatchVars struct { 46 46 - Transfers []string `json:"transfers,omitempty"` 47 47 - Operation string `json:"operation"` 48 48 - Objects []*RequestVars `json:"objects"` 49 49 - } 50 50 - 51 51 - // BatchResponse contains multiple object metadata Representation structures 52 52 - // for use with the batch API. 53 53 - type BatchResponse struct { 54 54 - Transfer string `json:"transfer,omitempty"` 55 55 - Objects []*Representation `json:"objects"` 56 56 - } 57 57 - 58 58 - // Representation is object metadata as seen by clients of the lfs server. 59 59 - type Representation struct { 60 60 - Oid string `json:"oid"` 61 61 - Size int64 `json:"size"` 62 62 - Actions map[string]*link `json:"actions"` 63 63 - Error *ObjectError `json:"error,omitempty"` 64 64 - } 65 65 - 66 66 - // ObjectError defines the JSON structure returned to the client in case of an error 67 67 - type ObjectError struct { 68 68 - Code int `json:"code"` 69 69 - Message string `json:"message"` 70 70 - } 71 71 - 72 34 // Claims is a JWT Token Claims 73 35 type Claims struct { 74 36 RepoID int64 ··· 78 40 } 79 41 80 42 // ObjectLink builds a URL linking to the object. 81 81 - func (v *RequestVars) ObjectLink() string { 82 82 - return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/objects", v.Oid) 43 43 + func (rc *requestContext) ObjectLink(oid string) string { 44 44 + return setting.AppURL + path.Join(rc.User, rc.Repo+".git", "info/lfs/objects", oid) 83 45 } 84 46 85 47 // VerifyLink builds a URL for verifying the object. 86 86 - func (v *RequestVars) VerifyLink() string { 87 87 - return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/verify") 88 88 - } 89 89 - 90 90 - // link provides a structure used to build a hypermedia representation of an HTTP link. 91 91 - type link struct { 92 92 - Href string `json:"href"` 93 93 - Header map[string]string `json:"header,omitempty"` 94 94 - ExpiresAt time.Time `json:"expires_at,omitempty"` 48 48 + func (rc *requestContext) VerifyLink() string { 49 49 + return setting.AppURL + path.Join(rc.User, rc.Repo+".git", "info/lfs/verify") 95 50 } 96 51 97 52 var oidRegExp = regexp.MustCompile(`^[A-Fa-f0-9]+$`) ··· 125 80 writeStatus(ctx, 404) 126 81 } 127 82 128 128 - func getAuthenticatedRepoAndMeta(ctx *context.Context, rv *RequestVars, requireWrite bool) (*models.LFSMetaObject, *models.Repository) { 129 129 - if !isOidValid(rv.Oid) { 130 130 - log.Info("Attempt to access invalid LFS OID[%s] in %s/%s", rv.Oid, rv.User, rv.Repo) 83 83 + func getAuthenticatedRepoAndMeta(ctx *context.Context, rc *requestContext, p lfs_module.Pointer, requireWrite bool) (*models.LFSMetaObject, *models.Repository) { 84 84 + if !isOidValid(p.Oid) { 85 85 + log.Info("Attempt to access invalid LFS OID[%s] in %s/%s", p.Oid, rc.User, rc.Repo) 131 86 writeStatus(ctx, 404) 132 87 return nil, nil 133 88 } 134 89 135 135 - repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo) 90 90 + repository, err := models.GetRepositoryByOwnerAndName(rc.User, rc.Repo) 136 91 if err != nil { 137 137 - log.Error("Unable to get repository: %s/%s Error: %v", rv.User, rv.Repo, err) 92 92 + log.Error("Unable to get repository: %s/%s Error: %v", rc.User, rc.Repo, err) 138 93 writeStatus(ctx, 404) 139 94 return nil, nil 140 95 } 141 96 142 142 - if !authenticate(ctx, repository, rv.Authorization, requireWrite) { 97 97 + if !authenticate(ctx, repository, rc.Authorization, requireWrite) { 143 98 requireAuth(ctx) 144 99 return nil, nil 145 100 } 146 101 147 147 - meta, err := repository.GetLFSMetaObjectByOid(rv.Oid) 102 102 + meta, err := repository.GetLFSMetaObjectByOid(p.Oid) 148 103 if err != nil { 149 149 - log.Error("Unable to get LFS OID[%s] Error: %v", rv.Oid, err) 104 104 + log.Error("Unable to get LFS OID[%s] Error: %v", p.Oid, err) 150 105 writeStatus(ctx, 404) 151 106 return nil, nil 152 107 } ··· 156 111 157 112 // getContentHandler gets the content from the content store 158 113 func getContentHandler(ctx *context.Context) { 159 159 - rv := unpack(ctx) 114 114 + rc, p := unpack(ctx) 160 115 161 161 - meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false) 116 116 + meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, false) 162 117 if meta == nil { 163 118 // Status already written in getAuthenticatedRepoAndMeta 164 119 return ··· 192 147 } 193 148 } 194 149 195 195 - contentStore := &ContentStore{ObjectStorage: storage.LFS} 196 196 - content, err := contentStore.Get(meta) 150 150 + contentStore := lfs_module.NewContentStore() 151 151 + content, err := contentStore.Get(meta.Pointer) 197 152 if err != nil { 198 153 // Errors are logged in contentStore.Get 199 154 writeStatus(ctx, http.StatusNotFound) ··· 233 188 234 189 // getMetaHandler retrieves metadata about the object 235 190 func getMetaHandler(ctx *context.Context) { 236 236 - rv := unpack(ctx) 191 191 + rc, p := unpack(ctx) 237 192 238 238 - meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false) 193 193 + meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, false) 239 194 if meta == nil { 240 195 // Status already written in getAuthenticatedRepoAndMeta 241 196 return 242 197 } 243 198 244 244 - ctx.Resp.Header().Set("Content-Type", metaMediaType) 199 199 + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) 245 200 246 201 if ctx.Req.Method == "GET" { 247 202 json := jsoniter.ConfigCompatibleWithStandardLibrary 248 203 enc := json.NewEncoder(ctx.Resp) 249 249 - if err := enc.Encode(Represent(rv, meta, true, false)); err != nil { 204 204 + if err := enc.Encode(represent(rc, meta.Pointer, true, false)); err != nil { 250 205 log.Error("Failed to encode representation as json. Error: %v", err) 251 206 } 252 207 } ··· 263 218 } 264 219 265 220 if !MetaMatcher(ctx.Req) { 266 266 - log.Info("Attempt to POST without accepting the correct media type: %s", metaMediaType) 221 221 + log.Info("Attempt to POST without accepting the correct media type: %s", lfs_module.MediaType) 267 222 writeStatus(ctx, 400) 268 223 return 269 224 } 270 225 271 271 - rv := unpack(ctx) 226 226 + rc, p := unpack(ctx) 272 227 273 273 - repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo) 228 228 + repository, err := models.GetRepositoryByOwnerAndName(rc.User, rc.Repo) 274 229 if err != nil { 275 275 - log.Error("Unable to get repository: %s/%s Error: %v", rv.User, rv.Repo, err) 230 230 + log.Error("Unable to get repository: %s/%s Error: %v", rc.User, rc.Repo, err) 276 231 writeStatus(ctx, 404) 277 232 return 278 233 } 279 234 280 280 - if !authenticate(ctx, repository, rv.Authorization, true) { 235 235 + if !authenticate(ctx, repository, rc.Authorization, true) { 281 236 requireAuth(ctx) 282 237 return 283 238 } 284 239 285 285 - if !isOidValid(rv.Oid) { 286 286 - log.Info("Invalid LFS OID[%s] attempt to POST in %s/%s", rv.Oid, rv.User, rv.Repo) 240 240 + if !isOidValid(p.Oid) { 241 241 + log.Info("Invalid LFS OID[%s] attempt to POST in %s/%s", p.Oid, rc.User, rc.Repo) 287 242 writeStatus(ctx, 404) 288 243 return 289 244 } 290 245 291 291 - if setting.LFS.MaxFileSize > 0 && rv.Size > setting.LFS.MaxFileSize { 292 292 - log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", rv.Oid, rv.Size, rv.User, rv.Repo, setting.LFS.MaxFileSize) 246 246 + if setting.LFS.MaxFileSize > 0 && p.Size > setting.LFS.MaxFileSize { 247 247 + log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", p.Oid, p.Size, rc.User, rc.Repo, setting.LFS.MaxFileSize) 293 248 writeStatus(ctx, 413) 294 249 return 295 250 } 296 251 297 297 - meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size, RepositoryID: repository.ID}) 252 252 + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: p, RepositoryID: repository.ID}) 298 253 if err != nil { 299 299 - log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", rv.Oid, rv.Size, rv.User, rv.Repo, err) 254 254 + log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", p.Oid, p.Size, rc.User, rc.Repo, err) 300 255 writeStatus(ctx, 404) 301 256 return 302 257 } 303 258 304 304 - ctx.Resp.Header().Set("Content-Type", metaMediaType) 259 259 + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) 305 260 306 261 sentStatus := 202 307 307 - contentStore := &ContentStore{ObjectStorage: storage.LFS} 308 308 - exist, err := contentStore.Exists(meta) 262 262 + contentStore := lfs_module.NewContentStore() 263 263 + exist, err := contentStore.Exists(p) 309 264 if err != nil { 310 310 - log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err) 265 265 + log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", p.Oid, rc.User, rc.Repo, err) 311 266 writeStatus(ctx, 500) 312 267 return 313 268 } ··· 318 273 319 274 json := jsoniter.ConfigCompatibleWithStandardLibrary 320 275 enc := json.NewEncoder(ctx.Resp) 321 321 - if err := enc.Encode(Represent(rv, meta, meta.Existing, true)); err != nil { 276 276 + if err := enc.Encode(represent(rc, meta.Pointer, meta.Existing, true)); err != nil { 322 277 log.Error("Failed to encode representation as json. Error: %v", err) 323 278 } 324 279 logRequest(ctx.Req, sentStatus) ··· 333 288 } 334 289 335 290 if !MetaMatcher(ctx.Req) { 336 336 - log.Info("Attempt to BATCH without accepting the correct media type: %s", metaMediaType) 291 291 + log.Info("Attempt to BATCH without accepting the correct media type: %s", lfs_module.MediaType) 337 292 writeStatus(ctx, 400) 338 293 return 339 294 } 340 295 341 296 bv := unpackbatch(ctx) 342 297 343 343 - var responseObjects []*Representation 298 298 + reqCtx := &requestContext{ 299 299 + User: ctx.Params("username"), 300 300 + Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"), 301 301 + Authorization: ctx.Req.Header.Get("Authorization"), 302 302 + } 303 303 + 304 304 + var responseObjects []*lfs_module.ObjectResponse 344 305 345 306 // Create a response object 346 307 for _, object := range bv.Objects { 347 308 if !isOidValid(object.Oid) { 348 348 - log.Info("Invalid LFS OID[%s] attempt to BATCH in %s/%s", object.Oid, object.User, object.Repo) 309 309 + log.Info("Invalid LFS OID[%s] attempt to BATCH in %s/%s", object.Oid, reqCtx.User, reqCtx.Repo) 349 310 continue 350 311 } 351 312 352 352 - repository, err := models.GetRepositoryByOwnerAndName(object.User, object.Repo) 313 313 + repository, err := models.GetRepositoryByOwnerAndName(reqCtx.User, reqCtx.Repo) 353 314 if err != nil { 354 354 - log.Error("Unable to get repository: %s/%s Error: %v", object.User, object.Repo, err) 315 315 + log.Error("Unable to get repository: %s/%s Error: %v", reqCtx.User, reqCtx.Repo, err) 355 316 writeStatus(ctx, 404) 356 317 return 357 318 } ··· 361 322 requireWrite = true 362 323 } 363 324 364 364 - if !authenticate(ctx, repository, object.Authorization, requireWrite) { 325 325 + if !authenticate(ctx, repository, reqCtx.Authorization, requireWrite) { 365 326 requireAuth(ctx) 366 327 return 367 328 } 368 329 369 369 - contentStore := &ContentStore{ObjectStorage: storage.LFS} 330 330 + contentStore := lfs_module.NewContentStore() 370 331 371 332 meta, err := repository.GetLFSMetaObjectByOid(object.Oid) 372 333 if err == nil { // Object is found and exists 373 373 - exist, err := contentStore.Exists(meta) 334 334 + exist, err := contentStore.Exists(meta.Pointer) 374 335 if err != nil { 375 375 - log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err) 336 336 + log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, reqCtx.User, reqCtx.Repo, err) 376 337 writeStatus(ctx, 500) 377 338 return 378 339 } 379 340 if exist { 380 380 - responseObjects = append(responseObjects, Represent(object, meta, true, false)) 341 341 + responseObjects = append(responseObjects, represent(reqCtx, meta.Pointer, true, false)) 381 342 continue 382 343 } 383 344 } 384 345 385 346 if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize { 386 386 - log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, object.User, object.Repo, setting.LFS.MaxFileSize) 347 347 + log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, reqCtx.User, reqCtx.Repo, setting.LFS.MaxFileSize) 387 348 writeStatus(ctx, 413) 388 349 return 389 350 } 390 351 391 352 // Object is not found 392 392 - meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID}) 353 353 + meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: object, RepositoryID: repository.ID}) 393 354 if err == nil { 394 394 - exist, err := contentStore.Exists(meta) 355 355 + exist, err := contentStore.Exists(meta.Pointer) 395 356 if err != nil { 396 396 - log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err) 357 357 + log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, reqCtx.User, reqCtx.Repo, err) 397 358 writeStatus(ctx, 500) 398 359 return 399 360 } 400 400 - responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !exist)) 361 361 + responseObjects = append(responseObjects, represent(reqCtx, meta.Pointer, meta.Existing, !exist)) 401 362 } else { 402 402 - log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err) 363 363 + log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, reqCtx.User, reqCtx.Repo, err) 403 364 } 404 365 } 405 366 406 406 - ctx.Resp.Header().Set("Content-Type", metaMediaType) 367 367 + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) 407 368 408 408 - respobj := &BatchResponse{Objects: responseObjects} 369 369 + respobj := &lfs_module.BatchResponse{Objects: responseObjects} 409 370 410 371 json := jsoniter.ConfigCompatibleWithStandardLibrary 411 372 enc := json.NewEncoder(ctx.Resp) ··· 417 378 418 379 // PutHandler receives data from the client and puts it into the content store 419 380 func PutHandler(ctx *context.Context) { 420 420 - rv := unpack(ctx) 381 381 + rc, p := unpack(ctx) 421 382 422 422 - meta, repository := getAuthenticatedRepoAndMeta(ctx, rv, true) 383 383 + meta, repository := getAuthenticatedRepoAndMeta(ctx, rc, p, true) 423 384 if meta == nil { 424 385 // Status already written in getAuthenticatedRepoAndMeta 425 386 return 426 387 } 427 388 428 428 - contentStore := &ContentStore{ObjectStorage: storage.LFS} 389 389 + contentStore := lfs_module.NewContentStore() 429 390 defer ctx.Req.Body.Close() 430 430 - if err := contentStore.Put(meta, ctx.Req.Body); err != nil { 391 391 + if err := contentStore.Put(meta.Pointer, ctx.Req.Body); err != nil { 431 392 // Put will log the error itself 432 393 ctx.Resp.WriteHeader(500) 433 433 - if err == errSizeMismatch || err == errHashMismatch { 394 394 + if err == lfs_module.ErrSizeMismatch || err == lfs_module.ErrHashMismatch { 434 395 fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err) 435 396 } else { 436 397 fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`) 437 398 } 438 438 - if _, err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil { 439 439 - log.Error("Whilst removing metaobject for LFS OID[%s] due to preceding error there was another Error: %v", rv.Oid, err) 399 399 + if _, err = repository.RemoveLFSMetaObjectByOid(p.Oid); err != nil { 400 400 + log.Error("Whilst removing metaobject for LFS OID[%s] due to preceding error there was another Error: %v", p.Oid, err) 440 401 } 441 402 return 442 403 } ··· 453 414 } 454 415 455 416 if !MetaMatcher(ctx.Req) { 456 456 - log.Info("Attempt to VERIFY without accepting the correct media type: %s", metaMediaType) 417 417 + log.Info("Attempt to VERIFY without accepting the correct media type: %s", lfs_module.MediaType) 457 418 writeStatus(ctx, 400) 458 419 return 459 420 } 460 421 461 461 - rv := unpack(ctx) 422 422 + rc, p := unpack(ctx) 462 423 463 463 - meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, true) 424 424 + meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, true) 464 425 if meta == nil { 465 426 // Status already written in getAuthenticatedRepoAndMeta 466 427 return 467 428 } 468 429 469 469 - contentStore := &ContentStore{ObjectStorage: storage.LFS} 470 470 - ok, err := contentStore.Verify(meta) 430 430 + contentStore := lfs_module.NewContentStore() 431 431 + ok, err := contentStore.Verify(meta.Pointer) 471 432 if err != nil { 472 433 // Error will be logged in Verify 473 434 ctx.Resp.WriteHeader(500) ··· 482 443 logRequest(ctx.Req, 200) 483 444 } 484 445 485 485 - // Represent takes a RequestVars and Meta and turns it into a Representation suitable 446 446 + // represent takes a requestContext and Meta and turns it into a ObjectResponse suitable 486 447 // for json encoding 487 487 - func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation { 488 488 - rep := &Representation{ 489 489 - Oid: meta.Oid, 490 490 - Size: meta.Size, 491 491 - Actions: make(map[string]*link), 448 448 + func represent(rc *requestContext, pointer lfs_module.Pointer, download, upload bool) *lfs_module.ObjectResponse { 449 449 + rep := &lfs_module.ObjectResponse{ 450 450 + Pointer: pointer, 451 451 + Actions: make(map[string]*lfs_module.Link), 492 452 } 493 453 494 454 header := make(map[string]string) 495 455 496 496 - if rv.Authorization == "" { 456 456 + if rc.Authorization == "" { 497 457 //https://github.com/github/git-lfs/issues/1088 498 458 header["Authorization"] = "Authorization: Basic dummy" 499 459 } else { 500 500 - header["Authorization"] = rv.Authorization 460 460 + header["Authorization"] = rc.Authorization 501 461 } 502 462 503 463 if download { 504 504 - rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header} 464 464 + rep.Actions["download"] = &lfs_module.Link{Href: rc.ObjectLink(pointer.Oid), Header: header} 505 465 } 506 466 507 467 if upload { 508 508 - rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header} 468 468 + rep.Actions["upload"] = &lfs_module.Link{Href: rc.ObjectLink(pointer.Oid), Header: header} 509 469 } 510 470 511 471 if upload && !download { ··· 516 476 } 517 477 518 478 // This is only needed to workaround https://github.com/git-lfs/git-lfs/issues/3662 519 519 - verifyHeader["Accept"] = metaMediaType 479 479 + verifyHeader["Accept"] = lfs_module.MediaType 520 480 521 521 - rep.Actions["verify"] = &link{Href: rv.VerifyLink(), Header: verifyHeader} 481 481 + rep.Actions["verify"] = &lfs_module.Link{Href: rc.VerifyLink(), Header: verifyHeader} 522 482 } 523 483 524 484 return rep 525 485 } 526 486 527 487 // MetaMatcher provides a mux.MatcherFunc that only allows requests that contain 528 528 - // an Accept header with the metaMediaType 488 488 + // an Accept header with the lfs_module.MediaType 529 489 func MetaMatcher(r *http.Request) bool { 530 490 mediaParts := strings.Split(r.Header.Get("Accept"), ";") 531 491 mt := mediaParts[0] 532 532 - return mt == metaMediaType 492 492 + return mt == lfs_module.MediaType 533 493 } 534 494 535 535 - func unpack(ctx *context.Context) *RequestVars { 495 495 + func unpack(ctx *context.Context) (*requestContext, lfs_module.Pointer) { 536 496 r := ctx.Req 537 537 - rv := &RequestVars{ 497 497 + rc := &requestContext{ 538 498 User: ctx.Params("username"), 539 499 Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"), 540 540 - Oid: ctx.Params("oid"), 541 500 Authorization: r.Header.Get("Authorization"), 542 501 } 502 502 + p := lfs_module.Pointer{Oid: ctx.Params("oid")} 543 503 544 504 if r.Method == "POST" { // Maybe also check if +json 545 545 - var p RequestVars 505 505 + var p2 lfs_module.Pointer 546 506 bodyReader := r.Body 547 507 defer bodyReader.Close() 548 508 json := jsoniter.ConfigCompatibleWithStandardLibrary 549 509 dec := json.NewDecoder(bodyReader) 550 550 - err := dec.Decode(&p) 510 510 + err := dec.Decode(&p2) 551 511 if err != nil { 552 512 // The error is logged as a WARN here because this may represent misbehaviour rather than a true error 553 553 - log.Warn("Unable to decode POST request vars for LFS OID[%s] in %s/%s: Error: %v", rv.Oid, rv.User, rv.Repo, err) 554 554 - return rv 513 513 + log.Warn("Unable to decode POST request vars for LFS OID[%s] in %s/%s: Error: %v", p.Oid, rc.User, rc.Repo, err) 514 514 + return rc, p 555 515 } 556 516 557 557 - rv.Oid = p.Oid 558 558 - rv.Size = p.Size 517 517 + p.Oid = p2.Oid 518 518 + p.Size = p2.Size 559 519 } 560 520 561 561 - return rv 521 521 + return rc, p 562 522 } 563 523 564 524 // TODO cheap hack, unify with unpack 565 565 - func unpackbatch(ctx *context.Context) *BatchVars { 525 525 + func unpackbatch(ctx *context.Context) *lfs_module.BatchRequest { 566 526 567 527 r := ctx.Req 568 568 - var bv BatchVars 528 528 + var bv lfs_module.BatchRequest 569 529 570 530 bodyReader := r.Body 571 531 defer bodyReader.Close() ··· 576 536 // The error is logged as a WARN here because this may represent misbehaviour rather than a true error 577 537 log.Warn("Unable to decode BATCH request vars in %s/%s: Error: %v", ctx.Params("username"), strings.TrimSuffix(ctx.Params("reponame"), ".git"), err) 578 538 return &bv 579 579 - } 580 580 - 581 581 - for i := 0; i < len(bv.Objects); i++ { 582 582 - bv.Objects[i].User = ctx.Params("username") 583 583 - bv.Objects[i].Repo = strings.TrimSuffix(ctx.Params("reponame"), ".git") 584 584 - bv.Objects[i].Authorization = r.Header.Get("Authorization") 585 539 } 586 540 587 541 return &bv

+69

modules/lfs/shared.go

reviewed

··· 1 1 + // Copyright 2020 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "time" 9 9 + ) 10 10 + 11 11 + const ( 12 12 + // MediaType contains the media type for LFS server requests 13 13 + MediaType = "application/vnd.git-lfs+json" 14 14 + ) 15 15 + 16 16 + // BatchRequest contains multiple requests processed in one batch operation. 17 17 + // https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#requests 18 18 + type BatchRequest struct { 19 19 + Operation string `json:"operation"` 20 20 + Transfers []string `json:"transfers,omitempty"` 21 21 + Ref *Reference `json:"ref,omitempty"` 22 22 + Objects []Pointer `json:"objects"` 23 23 + } 24 24 + 25 25 + // Reference contains a git reference. 26 26 + // https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#ref-property 27 27 + type Reference struct { 28 28 + Name string `json:"name"` 29 29 + } 30 30 + 31 31 + // Pointer contains LFS pointer data 32 32 + type Pointer struct { 33 33 + Oid string `json:"oid" xorm:"UNIQUE(s) INDEX NOT NULL"` 34 34 + Size int64 `json:"size" xorm:"NOT NULL"` 35 35 + } 36 36 + 37 37 + // BatchResponse contains multiple object metadata Representation structures 38 38 + // for use with the batch API. 39 39 + // https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#successful-responses 40 40 + type BatchResponse struct { 41 41 + Transfer string `json:"transfer,omitempty"` 42 42 + Objects []*ObjectResponse `json:"objects"` 43 43 + } 44 44 + 45 45 + // ObjectResponse is object metadata as seen by clients of the LFS server. 46 46 + type ObjectResponse struct { 47 47 + Pointer 48 48 + Actions map[string]*Link `json:"actions"` 49 49 + Error *ObjectError `json:"error,omitempty"` 50 50 + } 51 51 + 52 52 + // Link provides a structure used to build a hypermedia representation of an HTTP link. 53 53 + type Link struct { 54 54 + Href string `json:"href"` 55 55 + Header map[string]string `json:"header,omitempty"` 56 56 + ExpiresAt time.Time `json:"expires_at,omitempty"` 57 57 + } 58 58 + 59 59 + // ObjectError defines the JSON structure returned to the client in case of an error 60 60 + type ObjectError struct { 61 61 + Code int `json:"code"` 62 62 + Message string `json:"message"` 63 63 + } 64 64 + 65 65 + // PointerBlob associates a Git blob with a Pointer. 66 66 + type PointerBlob struct { 67 67 + Hash string 68 68 + Pointer 69 69 + }

+58

modules/lfs/transferadapter.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "context" 9 9 + "errors" 10 10 + "fmt" 11 11 + "io" 12 12 + "net/http" 13 13 + ) 14 14 + 15 15 + // TransferAdapter represents an adapter for downloading/uploading LFS objects 16 16 + type TransferAdapter interface { 17 17 + Name() string 18 18 + Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) 19 19 + //Upload(ctx context.Context, reader io.Reader) error 20 20 + } 21 21 + 22 22 + // BasicTransferAdapter implements the "basic" adapter 23 23 + type BasicTransferAdapter struct { 24 24 + client *http.Client 25 25 + } 26 26 + 27 27 + // Name returns the name of the adapter 28 28 + func (a *BasicTransferAdapter) Name() string { 29 29 + return "basic" 30 30 + } 31 31 + 32 32 + // Download reads the download location and downloads the data 33 33 + func (a *BasicTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { 34 34 + download, ok := r.Actions["download"] 35 35 + if !ok { 36 36 + return nil, errors.New("lfs.BasicTransferAdapter.Download: Action 'download' not found") 37 37 + } 38 38 + 39 39 + req, err := http.NewRequestWithContext(ctx, "GET", download.Href, nil) 40 40 + if err != nil { 41 41 + return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.NewRequestWithContext: %w", err) 42 42 + } 43 43 + for key, value := range download.Header { 44 44 + req.Header.Set(key, value) 45 45 + } 46 46 + 47 47 + res, err := a.client.Do(req) 48 48 + if err != nil { 49 49 + select { 50 50 + case <-ctx.Done(): 51 51 + return nil, ctx.Err() 52 52 + default: 53 53 + } 54 54 + return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.Do: %w", err) 55 55 + } 56 56 + 57 57 + return res.Body, nil 58 58 + }

+78

modules/lfs/transferadapter_test.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package lfs 6 6 + 7 7 + import ( 8 8 + "bytes" 9 9 + "context" 10 10 + "io/ioutil" 11 11 + "net/http" 12 12 + "strings" 13 13 + "testing" 14 14 + 15 15 + "github.com/stretchr/testify/assert" 16 16 + ) 17 17 + 18 18 + func TestBasicTransferAdapterName(t *testing.T) { 19 19 + a := &BasicTransferAdapter{} 20 20 + 21 21 + assert.Equal(t, "basic", a.Name()) 22 22 + } 23 23 + 24 24 + func TestBasicTransferAdapterDownload(t *testing.T) { 25 25 + roundTripHandler := func(req *http.Request) *http.Response { 26 26 + url := req.URL.String() 27 27 + if strings.Contains(url, "valid-download-request") { 28 28 + assert.Equal(t, "GET", req.Method) 29 29 + assert.Equal(t, "test-value", req.Header.Get("test-header")) 30 30 + 31 31 + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("dummy"))} 32 32 + } 33 33 + 34 34 + t.Errorf("Unknown test case: %s", url) 35 35 + 36 36 + return nil 37 37 + } 38 38 + 39 39 + hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)} 40 40 + a := &BasicTransferAdapter{hc} 41 41 + 42 42 + var cases = []struct { 43 43 + response *ObjectResponse 44 44 + expectederror string 45 45 + }{ 46 46 + // case 0 47 47 + { 48 48 + response: &ObjectResponse{}, 49 49 + expectederror: "Action 'download' not found", 50 50 + }, 51 51 + // case 1 52 52 + { 53 53 + response: &ObjectResponse{ 54 54 + Actions: map[string]*Link{"upload": nil}, 55 55 + }, 56 56 + expectederror: "Action 'download' not found", 57 57 + }, 58 58 + // case 2 59 59 + { 60 60 + response: &ObjectResponse{ 61 61 + Actions: map[string]*Link{"download": { 62 62 + Href: "https://valid-download-request.io", 63 63 + Header: map[string]string{"test-header": "test-value"}, 64 64 + }}, 65 65 + }, 66 66 + expectederror: "", 67 67 + }, 68 68 + } 69 69 + 70 70 + for n, c := range cases { 71 71 + _, err := a.Download(context.Background(), c.response) 72 72 + if len(c.expectederror) > 0 { 73 73 + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) 74 74 + } else { 75 75 + assert.NoError(t, err, "case %d", n) 76 76 + } 77 77 + } 78 78 + }

+2

modules/migrations/base/options.go

reviewed

··· 20 20 // required: true 21 21 RepoName string `json:"repo_name" binding:"Required"` 22 22 Mirror bool `json:"mirror"` 23 23 + LFS bool `json:"lfs"` 24 24 + LFSEndpoint string `json:"lfs_endpoint"` 23 25 Private bool `json:"private"` 24 26 Description string `json:"description"` 25 27 OriginalURL string

+2

modules/migrations/gitea_uploader.go

reviewed

··· 116 116 OriginalURL: repo.OriginalURL, 117 117 GitServiceType: opts.GitServiceType, 118 118 Mirror: repo.IsMirror, 119 119 + LFS: opts.LFS, 120 120 + LFSEndpoint: opts.LFSEndpoint, 119 121 CloneAddr: repo.CloneURL, 120 122 Private: repo.IsPrivate, 121 123 Wiki: opts.Wiki,

+6

modules/migrations/migrate.go

reviewed

··· 104 104 if err != nil { 105 105 return nil, err 106 106 } 107 107 + if opts.LFS && len(opts.LFSEndpoint) > 0 { 108 108 + err := IsMigrateURLAllowed(opts.LFSEndpoint, doer) 109 109 + if err != nil { 110 110 + return nil, err 111 111 + } 112 112 + } 107 113 downloader, err := newDownloader(ctx, ownerName, opts) 108 114 if err != nil { 109 115 return nil, err

+23 -25

modules/repofiles/update.go

reviewed

··· 18 18 "code.gitea.io/gitea/modules/log" 19 19 repo_module "code.gitea.io/gitea/modules/repository" 20 20 "code.gitea.io/gitea/modules/setting" 21 21 - "code.gitea.io/gitea/modules/storage" 22 21 "code.gitea.io/gitea/modules/structs" 23 22 24 23 stdcharset "golang.org/x/net/html/charset" ··· 70 69 buf = buf[:n] 71 70 72 71 if setting.LFS.StartServer { 73 73 - meta := lfs.IsPointerFile(&buf) 74 74 - if meta != nil { 75 75 - meta, err = repo.GetLFSMetaObjectByOid(meta.Oid) 72 72 + pointer, _ := lfs.ReadPointerFromBuffer(buf) 73 73 + if pointer.IsValid() { 74 74 + meta, err := repo.GetLFSMetaObjectByOid(pointer.Oid) 76 75 if err != nil && err != models.ErrLFSObjectNotExist { 77 76 // return default 78 77 return "UTF-8", false 79 78 } 80 80 - } 81 81 - if meta != nil { 82 82 - dataRc, err := lfs.ReadMetaObject(meta) 83 83 - if err != nil { 84 84 - // return default 85 85 - return "UTF-8", false 79 79 + if meta != nil { 80 80 + dataRc, err := lfs.ReadMetaObject(pointer) 81 81 + if err != nil { 82 82 + // return default 83 83 + return "UTF-8", false 84 84 + } 85 85 + defer dataRc.Close() 86 86 + buf = make([]byte, 1024) 87 87 + n, err = dataRc.Read(buf) 88 88 + if err != nil { 89 89 + // return default 90 90 + return "UTF-8", false 91 91 + } 92 92 + buf = buf[:n] 86 93 } 87 87 - defer dataRc.Close() 88 88 - buf = make([]byte, 1024) 89 89 - n, err = dataRc.Read(buf) 90 90 - if err != nil { 91 91 - // return default 92 92 - return "UTF-8", false 93 93 - } 94 94 - buf = buf[:n] 95 94 } 96 96 - 97 95 } 98 96 99 97 encoding, err := charset.DetectEncoding(buf) ··· 387 385 388 386 if filename2attribute2info[treePath] != nil && filename2attribute2info[treePath]["filter"] == "lfs" { 389 387 // OK so we are supposed to LFS this data! 390 390 - oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content)) 388 388 + pointer, err := lfs.GeneratePointer(strings.NewReader(opts.Content)) 391 389 if err != nil { 392 390 return nil, err 393 391 } 394 394 - lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID} 395 395 - content = lfsMetaObject.Pointer() 392 392 + lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: repo.ID} 393 393 + content = pointer.StringContent() 396 394 } 397 395 } 398 396 // Add the object to the database ··· 435 433 if err != nil { 436 434 return nil, err 437 435 } 438 438 - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} 439 439 - exist, err := contentStore.Exists(lfsMetaObject) 436 436 + contentStore := lfs.NewContentStore() 437 437 + exist, err := contentStore.Exists(lfsMetaObject.Pointer) 440 438 if err != nil { 441 439 return nil, err 442 440 } 443 441 if !exist { 444 444 - if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil { 442 442 + if err := contentStore.Put(lfsMetaObject.Pointer, strings.NewReader(opts.Content)); err != nil { 445 443 if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { 446 444 return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err) 447 445 }

+6 -11

modules/repofiles/upload.go

reviewed

··· 14 14 "code.gitea.io/gitea/modules/git" 15 15 "code.gitea.io/gitea/modules/lfs" 16 16 "code.gitea.io/gitea/modules/setting" 17 17 - "code.gitea.io/gitea/modules/storage" 18 17 ) 19 18 20 19 // UploadRepoFileOptions contains the uploaded repository file options ··· 137 136 138 137 // OK now we can insert the data into the store - there's no way to clean up the store 139 138 // once it's in there, it's in there. 140 140 - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} 139 139 + contentStore := lfs.NewContentStore() 141 140 for _, info := range infos { 142 141 if err := uploadToLFSContentStore(info, contentStore); err != nil { 143 142 return cleanUpAfterFailure(&infos, t, err) ··· 163 162 if setting.LFS.StartServer && filename2attribute2info[info.upload.Name] != nil && filename2attribute2info[info.upload.Name]["filter"] == "lfs" { 164 163 // Handle LFS 165 164 // FIXME: Inefficient! this should probably happen in models.Upload 166 166 - oid, err := models.GenerateLFSOid(file) 167 167 - if err != nil { 168 168 - return err 169 169 - } 170 170 - fileInfo, err := file.Stat() 165 165 + pointer, err := lfs.GeneratePointer(file) 171 166 if err != nil { 172 167 return err 173 168 } 174 169 175 175 - info.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID} 170 170 + info.lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: t.repo.ID} 176 171 177 177 - if objectHash, err = t.HashObject(strings.NewReader(info.lfsMetaObject.Pointer())); err != nil { 172 172 + if objectHash, err = t.HashObject(strings.NewReader(pointer.StringContent())); err != nil { 178 173 return err 179 174 } 180 175 } else if objectHash, err = t.HashObject(file); err != nil { ··· 189 184 if info.lfsMetaObject == nil { 190 185 return nil 191 186 } 192 192 - exist, err := contentStore.Exists(info.lfsMetaObject) 187 187 + exist, err := contentStore.Exists(info.lfsMetaObject.Pointer) 193 188 if err != nil { 194 189 return err 195 190 } ··· 202 197 defer file.Close() 203 198 // FIXME: Put regenerates the hash and copies the file over. 204 199 // I guess this strictly ensures the soundness of the store but this is inefficient. 205 205 - if err := contentStore.Put(info.lfsMetaObject, file); err != nil { 200 200 + if err := contentStore.Put(info.lfsMetaObject.Pointer, file); err != nil { 206 201 // OK Now we need to cleanup 207 202 // Can't clean up the store, once uploaded there they're there. 208 203 return err

+86

modules/repository/repo.go

reviewed

··· 7 7 import ( 8 8 "context" 9 9 "fmt" 10 10 + "net/url" 10 11 "path" 11 12 "strings" 12 13 "time" 13 14 14 15 "code.gitea.io/gitea/models" 15 16 "code.gitea.io/gitea/modules/git" 17 17 + "code.gitea.io/gitea/modules/lfs" 16 18 "code.gitea.io/gitea/modules/log" 17 19 migration "code.gitea.io/gitea/modules/migrations/base" 18 20 "code.gitea.io/gitea/modules/setting" ··· 120 122 log.Error("Failed to synchronize tags to releases for repository: %v", err) 121 123 } 122 124 } 125 125 + 126 126 + if opts.LFS { 127 127 + ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) 128 128 + if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep); err != nil { 129 129 + log.Error("Failed to store missing LFS objects for repository: %v", err) 130 130 + } 131 131 + } 123 132 } 124 133 125 134 if err = repo.UpdateSize(models.DefaultDBContext()); err != nil { ··· 132 141 Interval: setting.Mirror.DefaultInterval, 133 142 EnablePrune: true, 134 143 NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), 144 144 + LFS: opts.LFS, 145 145 + } 146 146 + if opts.LFS { 147 147 + mirrorModel.LFSEndpoint = opts.LFSEndpoint 135 148 } 136 149 137 150 if opts.MirrorInterval != "" { ··· 300 313 301 314 return models.SaveOrUpdateTag(repo, &rel) 302 315 } 316 316 + 317 317 + // StoreMissingLfsObjectsInRepository downloads missing LFS objects 318 318 + func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL) error { 319 319 + client := lfs.NewClient(endpoint) 320 320 + contentStore := lfs.NewContentStore() 321 321 + 322 322 + pointerChan := make(chan lfs.PointerBlob) 323 323 + errChan := make(chan error, 1) 324 324 + go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) 325 325 + 326 326 + err := func() error { 327 327 + for pointerBlob := range pointerChan { 328 328 + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: pointerBlob.Pointer, RepositoryID: repo.ID}) 329 329 + if err != nil { 330 330 + return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err) 331 331 + } 332 332 + if meta.Existing { 333 333 + continue 334 334 + } 335 335 + 336 336 + log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] not present in repository %s", pointerBlob.Oid, repo.FullName()) 337 337 + 338 338 + err = func() error { 339 339 + exist, err := contentStore.Exists(pointerBlob.Pointer) 340 340 + if err != nil { 341 341 + return fmt.Errorf("StoreMissingLfsObjectsInRepository contentStore.Exists: %w", err) 342 342 + } 343 343 + if !exist { 344 344 + if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize { 345 345 + log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Oid, setting.LFS.MaxFileSize, pointerBlob.Size) 346 346 + return nil 347 347 + } 348 348 + 349 349 + stream, err := client.Download(ctx, pointerBlob.Oid, pointerBlob.Size) 350 350 + if err != nil { 351 351 + return fmt.Errorf("StoreMissingLfsObjectsInRepository: LFS OID[%s] failed to download: %w", pointerBlob.Oid, err) 352 352 + } 353 353 + defer stream.Close() 354 354 + 355 355 + if err := contentStore.Put(pointerBlob.Pointer, stream); err != nil { 356 356 + return fmt.Errorf("StoreMissingLfsObjectsInRepository LFS OID[%s] contentStore.Put: %w", pointerBlob.Oid, err) 357 357 + } 358 358 + } else { 359 359 + log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] already present in content store", pointerBlob.Oid) 360 360 + } 361 361 + return nil 362 362 + }() 363 363 + if err != nil { 364 364 + if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { 365 365 + log.Error("StoreMissingLfsObjectsInRepository RemoveLFSMetaObjectByOid[Oid: %s]: %w", meta.Oid, err2) 366 366 + } 367 367 + 368 368 + select { 369 369 + case <-ctx.Done(): 370 370 + return nil 371 371 + default: 372 372 + } 373 373 + return err 374 374 + } 375 375 + } 376 376 + return nil 377 377 + }() 378 378 + if err != nil { 379 379 + return err 380 380 + } 381 381 + 382 382 + err, has := <-errChan 383 383 + if has { 384 384 + return err 385 385 + } 386 386 + 387 387 + return nil 388 388 + }

+2

modules/structs/repo.go

reviewed

··· 260 260 AuthToken string `json:"auth_token"` 261 261 262 262 Mirror bool `json:"mirror"` 263 263 + LFS bool `json:"lfs"` 264 264 + LFSEndpoint string `json:"lfs_endpoint"` 263 265 Private bool `json:"private"` 264 266 Description string `json:"description" binding:"MaxSize(255)"` 265 267 Wiki bool `json:"wiki"`

+23

modules/util/path.go

reviewed

··· 6 6 7 7 import ( 8 8 "errors" 9 9 + "net/url" 9 10 "os" 10 11 "path" 11 12 "path/filepath" 13 13 + "regexp" 14 14 + "runtime" 12 15 "strings" 13 16 ) 14 17 ··· 150 153 } 151 154 return statDir(rootPath, "", isIncludeDir, false, false) 152 155 } 156 156 + 157 157 + // FileURLToPath extracts the path informations from a file://... url. 158 158 + func FileURLToPath(u *url.URL) (string, error) { 159 159 + if u.Scheme != "file" { 160 160 + return "", errors.New("URL scheme is not 'file': " + u.String()) 161 161 + } 162 162 + 163 163 + path := u.Path 164 164 + 165 165 + if runtime.GOOS != "windows" { 166 166 + return path, nil 167 167 + } 168 168 + 169 169 + // If it looks like there's a Windows drive letter at the beginning, strip off the leading slash. 170 170 + re := regexp.MustCompile("/[A-Za-z]:/") 171 171 + if re.MatchString(path) { 172 172 + return path[1:], nil 173 173 + } 174 174 + return path, nil 175 175 + }

+58

modules/util/path_test.go

reviewed

··· 1 1 + // Copyright 2021 The Gitea Authors. All rights reserved. 2 2 + // Use of this source code is governed by a MIT-style 3 3 + // license that can be found in the LICENSE file. 4 4 + 5 5 + package util 6 6 + 7 7 + import ( 8 8 + "net/url" 9 9 + "runtime" 10 10 + "testing" 11 11 + 12 12 + "github.com/stretchr/testify/assert" 13 13 + ) 14 14 + 15 15 + func TestFileURLToPath(t *testing.T) { 16 16 + var cases = []struct { 17 17 + url string 18 18 + expected string 19 19 + haserror bool 20 20 + windows bool 21 21 + }{ 22 22 + // case 0 23 23 + { 24 24 + url: "", 25 25 + haserror: true, 26 26 + }, 27 27 + // case 1 28 28 + { 29 29 + url: "http://test.io", 30 30 + haserror: true, 31 31 + }, 32 32 + // case 2 33 33 + { 34 34 + url: "file:///path", 35 35 + expected: "/path", 36 36 + }, 37 37 + // case 3 38 38 + { 39 39 + url: "file:///C:/path", 40 40 + expected: "C:/path", 41 41 + windows: true, 42 42 + }, 43 43 + } 44 44 + 45 45 + for n, c := range cases { 46 46 + if c.windows && runtime.GOOS != "windows" { 47 47 + continue 48 48 + } 49 49 + u, _ := url.Parse(c.url) 50 50 + p, err := FileURLToPath(u) 51 51 + if c.haserror { 52 52 + assert.Error(t, err, "case %d: should return error", n) 53 53 + } else { 54 54 + assert.NoError(t, err, "case %d: should not return error", n) 55 55 + assert.Equal(t, c.expected, p, "case %d: should be equal", n) 56 56 + } 57 57 + } 58 58 + }

+10 -1

options/locale/locale_en-US.ini

reviewed

··· 726 726 mirror_address_desc = Put any required credentials in the Clone Authorization section. 727 727 mirror_address_url_invalid = The provided url is invalid. You must escape all components of the url correctly. 728 728 mirror_address_protocol_invalid = The provided url is invalid. Only http(s):// or git:// locations can be mirrored from. 729 729 + mirror_lfs = Large File System (LFS) 730 730 + mirror_lfs_desc = Activate mirroring of LFS data. 731 731 + mirror_lfs_endpoint = LFS Endpoint 732 732 + mirror_lfs_endpoint_desc = Sync will attempt to use the clone url to <a target="_blank" rel="noopener noreferrer" href="%s">determine the LFS server</a>. You can also specify a custom endpoint if the repository LFS data is stored somewhere else. 729 733 mirror_last_synced = Last Synchronized 730 734 watchers = Watchers 731 735 stargazers = Stargazers ··· 784 788 migrate_service = Migration Service 785 789 migrate_options_mirror_helper = This repository will be a <span class="text blue">mirror</span> 786 790 migrate_options_mirror_disabled = Your site administrator has disabled new mirrors. 791 791 + migrate_options_lfs = Migrate LFS files 792 792 + migrate_options_lfs_endpoint.label = LFS Endpoint 793 793 + migrate_options_lfs_endpoint.description = Migration will attempt to use your Git remote to <a target="_blank" rel="noopener noreferrer" href="%s">determine the LFS server</a>. You can also specify a custom endpoint if the repository LFS data is stored somewhere else. 794 794 + migrate_options_lfs_endpoint.description.local = A local server path is supported too. 795 795 + migrate_options_lfs_endpoint.placeholder = Leave blank to derive from clone URL 787 796 migrate_items = Migration Items 788 797 migrate_items_wiki = Wiki 789 798 migrate_items_milestones = Milestones ··· 800 809 migrate.permission_denied_blocked = You are not allowed to import from blocked hosts. 801 810 migrate.permission_denied_private_ip = You are not allowed to import from private IPs. 802 811 migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory." 812 812 + migrate.invalid_lfs_endpoint = The LFS endpoint is not valid. 803 813 migrate.failed = Migration failed: %v 804 804 - migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'git lfs fetch --all' and 'git lfs push --all' instead. 805 814 migrate.migrate_items_options = Access Token is required to migrate additional items 806 815 migrated_from = Migrated from <a href="%[1]s">%[2]s</a> 807 816 migrated_from_fake = Migrated From %[1]s

+43 -21

routers/api/v1/repo/migrate.go

reviewed

··· 15 15 "code.gitea.io/gitea/modules/context" 16 16 "code.gitea.io/gitea/modules/convert" 17 17 "code.gitea.io/gitea/modules/graceful" 18 18 + "code.gitea.io/gitea/modules/lfs" 18 19 "code.gitea.io/gitea/modules/log" 19 20 "code.gitea.io/gitea/modules/migrations" 20 21 "code.gitea.io/gitea/modules/migrations/base" ··· 101 102 err = migrations.IsMigrateURLAllowed(remoteAddr, ctx.User) 102 103 } 103 104 if err != nil { 104 104 - if models.IsErrInvalidCloneAddr(err) { 105 105 - addrErr := err.(*models.ErrInvalidCloneAddr) 106 106 - switch { 107 107 - case addrErr.IsURLError: 108 108 - ctx.Error(http.StatusUnprocessableEntity, "", err) 109 109 - case addrErr.IsPermissionDenied: 110 110 - if addrErr.LocalPath { 111 111 - ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import local repositories.") 112 112 - } else if len(addrErr.PrivateNet) == 0 { 113 113 - ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from blocked hosts.") 114 114 - } else { 115 115 - ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from private IPs.") 116 116 - } 117 117 - case addrErr.IsInvalidPath: 118 118 - ctx.Error(http.StatusUnprocessableEntity, "", "Invalid local path, it does not exist or not a directory.") 119 119 - default: 120 120 - ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", "Unknown error type (ErrInvalidCloneAddr): "+err.Error()) 121 121 - } 122 122 - } else { 123 123 - ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", err) 124 124 - } 105 105 + handleRemoteAddrError(ctx, err) 125 106 return 126 107 } 127 108 ··· 137 118 return 138 119 } 139 120 121 121 + form.LFS = form.LFS && setting.LFS.StartServer 122 122 + 123 123 + if form.LFS && len(form.LFSEndpoint) > 0 { 124 124 + ep := lfs.DetermineEndpoint("", form.LFSEndpoint) 125 125 + if ep == nil { 126 126 + ctx.Error(http.StatusInternalServerError, "", ctx.Tr("repo.migrate.invalid_lfs_endpoint")) 127 127 + return 128 128 + } 129 129 + err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User) 130 130 + if err != nil { 131 131 + handleRemoteAddrError(ctx, err) 132 132 + return 133 133 + } 134 134 + } 135 135 + 140 136 var opts = migrations.MigrateOptions{ 141 137 CloneAddr: remoteAddr, 142 138 RepoName: form.RepoName, 143 139 Description: form.Description, 144 140 Private: form.Private || setting.Repository.ForcePrivate, 145 141 Mirror: form.Mirror, 142 142 + LFS: form.LFS, 143 143 + LFSEndpoint: form.LFSEndpoint, 146 144 AuthUsername: form.AuthUsername, 147 145 AuthPassword: form.AuthPassword, 148 146 AuthToken: form.AuthToken, ··· 245 243 } 246 244 } 247 245 } 246 246 + 247 247 + func handleRemoteAddrError(ctx *context.APIContext, err error) { 248 248 + if models.IsErrInvalidCloneAddr(err) { 249 249 + addrErr := err.(*models.ErrInvalidCloneAddr) 250 250 + switch { 251 251 + case addrErr.IsURLError: 252 252 + ctx.Error(http.StatusUnprocessableEntity, "", err) 253 253 + case addrErr.IsPermissionDenied: 254 254 + if addrErr.LocalPath { 255 255 + ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import local repositories.") 256 256 + } else if len(addrErr.PrivateNet) == 0 { 257 257 + ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from blocked hosts.") 258 258 + } else { 259 259 + ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from private IPs.") 260 260 + } 261 261 + case addrErr.IsInvalidPath: 262 262 + ctx.Error(http.StatusUnprocessableEntity, "", "Invalid local path, it does not exist or not a directory.") 263 263 + default: 264 264 + ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", "Unknown error type (ErrInvalidCloneAddr): "+err.Error()) 265 265 + } 266 266 + } else { 267 267 + ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", err) 268 268 + } 269 269 + }

+4 -3

routers/repo/download.go

reviewed

··· 96 96 } 97 97 }() 98 98 99 99 - if meta, _ := lfs.ReadPointerFile(dataRc); meta != nil { 100 100 - meta, _ = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid) 99 99 + pointer, _ := lfs.ReadPointer(dataRc) 100 100 + if pointer.IsValid() { 101 101 + meta, _ := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) 101 102 if meta == nil { 102 103 return ServeBlob(ctx, blob) 103 104 } 104 104 - lfsDataRc, err := lfs.ReadMetaObject(meta) 105 105 + lfsDataRc, err := lfs.ReadMetaObject(meta.Pointer) 105 106 if err != nil { 106 107 return err 107 108 }

+74 -136

routers/repo/lfs.go

reviewed

··· 5 5 package repo 6 6 7 7 import ( 8 8 - "bufio" 9 8 "bytes" 10 9 "fmt" 11 10 gotemplate "html/template" ··· 15 14 "path" 16 15 "strconv" 17 16 "strings" 18 18 - "sync" 19 17 20 18 "code.gitea.io/gitea/models" 21 19 "code.gitea.io/gitea/modules/base" ··· 266 264 return 267 265 } 268 266 ctx.Data["LFSFile"] = meta 269 269 - dataRc, err := lfs.ReadMetaObject(meta) 267 267 + dataRc, err := lfs.ReadMetaObject(meta.Pointer) 270 268 if err != nil { 271 269 ctx.ServerError("LFSFileGet", err) 272 270 return ··· 385 383 ctx.Data["PageIsSettingsLFS"] = true 386 384 var hash git.SHA1 387 385 if len(sha) == 0 { 388 388 - meta := models.LFSMetaObject{Oid: oid, Size: size} 389 389 - pointer := meta.Pointer() 390 390 - hash = git.ComputeBlobHash([]byte(pointer)) 386 386 + pointer := lfs.Pointer{Oid: oid, Size: size} 387 387 + hash = git.ComputeBlobHash([]byte(pointer.StringContent())) 391 388 sha = hash.String() 392 389 } else { 393 390 hash = git.MustIDFromString(sha) ··· 421 418 } 422 419 ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" 423 420 424 424 - basePath := ctx.Repo.Repository.RepoPath() 421 421 + err = func() error { 422 422 + pointerChan := make(chan lfs.PointerBlob) 423 423 + errChan := make(chan error, 1) 424 424 + go lfs.SearchPointerBlobs(ctx.Req.Context(), ctx.Repo.GitRepo, pointerChan, errChan) 425 425 426 426 - pointerChan := make(chan pointerResult) 426 426 + numPointers := 0 427 427 + var numAssociated, numNoExist, numAssociatable int 427 428 428 428 - catFileCheckReader, catFileCheckWriter := io.Pipe() 429 429 - shasToBatchReader, shasToBatchWriter := io.Pipe() 430 430 - catFileBatchReader, catFileBatchWriter := io.Pipe() 431 431 - errChan := make(chan error, 1) 432 432 - wg := sync.WaitGroup{} 433 433 - wg.Add(5) 429 429 + type pointerResult struct { 430 430 + SHA string 431 431 + Oid string 432 432 + Size int64 433 433 + InRepo bool 434 434 + Exists bool 435 435 + Accessible bool 436 436 + } 437 437 + 438 438 + results := []pointerResult{} 439 439 + 440 440 + contentStore := lfs.NewContentStore() 441 441 + repo := ctx.Repo.Repository 442 442 + 443 443 + for pointerBlob := range pointerChan { 444 444 + numPointers++ 445 445 + 446 446 + result := pointerResult{ 447 447 + SHA: pointerBlob.Hash, 448 448 + Oid: pointerBlob.Oid, 449 449 + Size: pointerBlob.Size, 450 450 + } 451 451 + 452 452 + if _, err := repo.GetLFSMetaObjectByOid(pointerBlob.Oid); err != nil { 453 453 + if err != models.ErrLFSObjectNotExist { 454 454 + return err 455 455 + } 456 456 + } else { 457 457 + result.InRepo = true 458 458 + } 459 459 + 460 460 + result.Exists, err = contentStore.Exists(pointerBlob.Pointer) 461 461 + if err != nil { 462 462 + return err 463 463 + } 434 464 435 435 - var numPointers, numAssociated, numNoExist, numAssociatable int 465 465 + if result.Exists { 466 466 + if !result.InRepo { 467 467 + // Can we fix? 468 468 + // OK well that's "simple" 469 469 + // - we need to check whether current user has access to a repo that has access to the file 470 470 + result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid) 471 471 + if err != nil { 472 472 + return err 473 473 + } 474 474 + } else { 475 475 + result.Accessible = true 476 476 + } 477 477 + } 436 478 437 437 - go func() { 438 438 - defer wg.Done() 439 439 - pointers := make([]pointerResult, 0, 50) 440 440 - for pointer := range pointerChan { 441 441 - pointers = append(pointers, pointer) 442 442 - if pointer.InRepo { 479 479 + if result.InRepo { 443 480 numAssociated++ 444 481 } 445 445 - if !pointer.Exists { 482 482 + if !result.Exists { 446 483 numNoExist++ 447 484 } 448 448 - if !pointer.InRepo && pointer.Accessible { 485 485 + if !result.InRepo && result.Accessible { 449 486 numAssociatable++ 450 487 } 488 488 + 489 489 + results = append(results, result) 451 490 } 452 452 - numPointers = len(pointers) 453 453 - ctx.Data["Pointers"] = pointers 491 491 + 492 492 + err, has := <-errChan 493 493 + if has { 494 494 + return err 495 495 + } 496 496 + 497 497 + ctx.Data["Pointers"] = results 454 498 ctx.Data["NumPointers"] = numPointers 455 499 ctx.Data["NumAssociated"] = numAssociated 456 500 ctx.Data["NumAssociatable"] = numAssociatable 457 501 ctx.Data["NumNoExist"] = numNoExist 458 502 ctx.Data["NumNotAssociated"] = numPointers - numAssociated 503 503 + 504 504 + return nil 459 505 }() 460 460 - go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User) 461 461 - go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) 462 462 - go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) 463 463 - if git.CheckGitVersionAtLeast("2.6.0") != nil { 464 464 - revListReader, revListWriter := io.Pipe() 465 465 - shasToCheckReader, shasToCheckWriter := io.Pipe() 466 466 - wg.Add(2) 467 467 - go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath) 468 468 - go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg) 469 469 - go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan) 470 470 - } else { 471 471 - go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan) 506 506 + if err != nil { 507 507 + ctx.ServerError("LFSPointerFiles", err) 508 508 + return 472 509 } 473 473 - wg.Wait() 474 510 475 475 - select { 476 476 - case err, has := <-errChan: 477 477 - if has { 478 478 - ctx.ServerError("LFSPointerFiles", err) 479 479 - } 480 480 - default: 481 481 - } 482 511 ctx.HTML(http.StatusOK, tplSettingsLFSPointers) 483 483 - } 484 484 - 485 485 - type pointerResult struct { 486 486 - SHA string 487 487 - Oid string 488 488 - Size int64 489 489 - InRepo bool 490 490 - Exists bool 491 491 - Accessible bool 492 492 - } 493 493 - 494 494 - func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { 495 495 - defer wg.Done() 496 496 - defer catFileBatchReader.Close() 497 497 - contentStore := lfs.ContentStore{ObjectStorage: storage.LFS} 498 498 - 499 499 - bufferedReader := bufio.NewReader(catFileBatchReader) 500 500 - buf := make([]byte, 1025) 501 501 - for { 502 502 - // File descriptor line: sha 503 503 - sha, err := bufferedReader.ReadString(' ') 504 504 - if err != nil { 505 505 - _ = catFileBatchReader.CloseWithError(err) 506 506 - break 507 507 - } 508 508 - // Throw away the blob 509 509 - if _, err := bufferedReader.ReadString(' '); err != nil { 510 510 - _ = catFileBatchReader.CloseWithError(err) 511 511 - break 512 512 - } 513 513 - sizeStr, err := bufferedReader.ReadString('\n') 514 514 - if err != nil { 515 515 - _ = catFileBatchReader.CloseWithError(err) 516 516 - break 517 517 - } 518 518 - size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1]) 519 519 - if err != nil { 520 520 - _ = catFileBatchReader.CloseWithError(err) 521 521 - break 522 522 - } 523 523 - pointerBuf := buf[:size+1] 524 524 - if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil { 525 525 - _ = catFileBatchReader.CloseWithError(err) 526 526 - break 527 527 - } 528 528 - pointerBuf = pointerBuf[:size] 529 529 - // Now we need to check if the pointerBuf is an LFS pointer 530 530 - pointer := lfs.IsPointerFile(&pointerBuf) 531 531 - if pointer == nil { 532 532 - continue 533 533 - } 534 534 - 535 535 - result := pointerResult{ 536 536 - SHA: strings.TrimSpace(sha), 537 537 - Oid: pointer.Oid, 538 538 - Size: pointer.Size, 539 539 - } 540 540 - 541 541 - // Then we need to check that this pointer is in the db 542 542 - if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil { 543 543 - if err != models.ErrLFSObjectNotExist { 544 544 - _ = catFileBatchReader.CloseWithError(err) 545 545 - break 546 546 - } 547 547 - } else { 548 548 - result.InRepo = true 549 549 - } 550 550 - 551 551 - result.Exists, err = contentStore.Exists(pointer) 552 552 - if err != nil { 553 553 - _ = catFileBatchReader.CloseWithError(err) 554 554 - break 555 555 - } 556 556 - 557 557 - if result.Exists { 558 558 - if !result.InRepo { 559 559 - // Can we fix? 560 560 - // OK well that's "simple" 561 561 - // - we need to check whether current user has access to a repo that has access to the file 562 562 - result.Accessible, err = models.LFSObjectAccessible(user, result.Oid) 563 563 - if err != nil { 564 564 - _ = catFileBatchReader.CloseWithError(err) 565 565 - break 566 566 - } 567 567 - } else { 568 568 - result.Accessible = true 569 569 - } 570 570 - } 571 571 - pointerChan <- result 572 572 - } 573 573 - close(pointerChan) 574 512 } 575 513 576 514 // LFSAutoAssociate auto associates accessible lfs files

+51 -26

routers/repo/migrate.go

reviewed

··· 12 12 "code.gitea.io/gitea/models" 13 13 "code.gitea.io/gitea/modules/base" 14 14 "code.gitea.io/gitea/modules/context" 15 15 + "code.gitea.io/gitea/modules/lfs" 15 16 "code.gitea.io/gitea/modules/log" 16 17 "code.gitea.io/gitea/modules/migrations" 17 18 "code.gitea.io/gitea/modules/setting" ··· 47 48 48 49 ctx.Data["private"] = getRepoPrivate(ctx) 49 50 ctx.Data["mirror"] = ctx.Query("mirror") == "1" 51 51 + ctx.Data["lfs"] = ctx.Query("lfs") == "1" 50 52 ctx.Data["wiki"] = ctx.Query("wiki") == "1" 51 53 ctx.Data["milestones"] = ctx.Query("milestones") == "1" 52 54 ctx.Data["labels"] = ctx.Query("labels") == "1" ··· 114 116 } 115 117 } 116 118 119 119 + func handleMigrateRemoteAddrError(ctx *context.Context, err error, tpl base.TplName, form *forms.MigrateRepoForm) { 120 120 + if models.IsErrInvalidCloneAddr(err) { 121 121 + addrErr := err.(*models.ErrInvalidCloneAddr) 122 122 + switch { 123 123 + case addrErr.IsProtocolInvalid: 124 124 + ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tpl, form) 125 125 + case addrErr.IsURLError: 126 126 + ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form) 127 127 + case addrErr.IsPermissionDenied: 128 128 + if addrErr.LocalPath { 129 129 + ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, form) 130 130 + } else if len(addrErr.PrivateNet) == 0 { 131 131 + ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, form) 132 132 + } else { 133 133 + ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tpl, form) 134 134 + } 135 135 + case addrErr.IsInvalidPath: 136 136 + ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, form) 137 137 + default: 138 138 + log.Error("Error whilst updating url: %v", err) 139 139 + ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form) 140 140 + } 141 141 + } else { 142 142 + log.Error("Error whilst updating url: %v", err) 143 143 + ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form) 144 144 + } 145 145 + } 146 146 + 117 147 // MigratePost response for migrating from external git repository 118 148 func MigratePost(ctx *context.Context) { 119 149 form := web.GetForm(ctx).(*forms.MigrateRepoForm) ··· 144 174 err = migrations.IsMigrateURLAllowed(remoteAddr, ctx.User) 145 175 } 146 176 if err != nil { 147 147 - if models.IsErrInvalidCloneAddr(err) { 148 148 - ctx.Data["Err_CloneAddr"] = true 149 149 - addrErr := err.(*models.ErrInvalidCloneAddr) 150 150 - switch { 151 151 - case addrErr.IsProtocolInvalid: 152 152 - ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tpl, &form) 153 153 - case addrErr.IsURLError: 154 154 - ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, &form) 155 155 - case addrErr.IsPermissionDenied: 156 156 - if addrErr.LocalPath { 157 157 - ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, &form) 158 158 - } else if len(addrErr.PrivateNet) == 0 { 159 159 - ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, &form) 160 160 - } else { 161 161 - ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tpl, &form) 162 162 - } 163 163 - case addrErr.IsInvalidPath: 164 164 - ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, &form) 165 165 - default: 166 166 - log.Error("Error whilst updating url: %v", err) 167 167 - ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, &form) 168 168 - } 169 169 - } else { 170 170 - log.Error("Error whilst updating url: %v", err) 171 171 - ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, &form) 177 177 + ctx.Data["Err_CloneAddr"] = true 178 178 + handleMigrateRemoteAddrError(ctx, err, tpl, form) 179 179 + return 180 180 + } 181 181 + 182 182 + form.LFS = form.LFS && setting.LFS.StartServer 183 183 + 184 184 + if form.LFS && len(form.LFSEndpoint) > 0 { 185 185 + ep := lfs.DetermineEndpoint("", form.LFSEndpoint) 186 186 + if ep == nil { 187 187 + ctx.Data["Err_LFSEndpoint"] = true 188 188 + ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_lfs_endpoint"), tpl, &form) 189 189 + return 190 190 + } 191 191 + err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User) 192 192 + if err != nil { 193 193 + ctx.Data["Err_LFSEndpoint"] = true 194 194 + handleMigrateRemoteAddrError(ctx, err, tpl, form) 195 195 + return 172 196 } 173 173 - return 174 197 } 175 198 176 199 var opts = migrations.MigrateOptions{ ··· 181 204 Description: form.Description, 182 205 Private: form.Private || setting.Repository.ForcePrivate, 183 206 Mirror: form.Mirror && !setting.Repository.DisableMirrors, 207 207 + LFS: form.LFS, 208 208 + LFSEndpoint: form.LFSEndpoint, 184 209 AuthUsername: form.AuthUsername, 185 210 AuthPassword: form.AuthPassword, 186 211 AuthToken: form.AuthToken,

+51 -23

routers/repo/setting.go

reviewed

··· 17 17 "code.gitea.io/gitea/modules/base" 18 18 "code.gitea.io/gitea/modules/context" 19 19 "code.gitea.io/gitea/modules/git" 20 20 + "code.gitea.io/gitea/modules/lfs" 20 21 "code.gitea.io/gitea/modules/log" 21 22 "code.gitea.io/gitea/modules/migrations" 22 23 "code.gitea.io/gitea/modules/repository" ··· 170 171 err = migrations.IsMigrateURLAllowed(address, ctx.User) 171 172 } 172 173 if err != nil { 173 173 - if models.IsErrInvalidCloneAddr(err) { 174 174 - ctx.Data["Err_MirrorAddress"] = true 175 175 - addrErr := err.(*models.ErrInvalidCloneAddr) 176 176 - switch { 177 177 - case addrErr.IsProtocolInvalid: 178 178 - ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tplSettingsOptions, &form) 179 179 - case addrErr.IsURLError: 180 180 - ctx.RenderWithErr(ctx.Tr("form.url_error"), tplSettingsOptions, &form) 181 181 - case addrErr.IsPermissionDenied: 182 182 - if addrErr.LocalPath { 183 183 - ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, &form) 184 184 - } else if len(addrErr.PrivateNet) == 0 { 185 185 - ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, &form) 186 186 - } else { 187 187 - ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tplSettingsOptions, &form) 188 188 - } 189 189 - case addrErr.IsInvalidPath: 190 190 - ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, &form) 191 191 - default: 192 192 - ctx.ServerError("Unknown error", err) 193 193 - } 194 194 - } 195 174 ctx.Data["Err_MirrorAddress"] = true 196 196 - ctx.RenderWithErr(ctx.Tr("repo.mirror_address_url_invalid"), tplSettingsOptions, &form) 175 175 + handleSettingRemoteAddrError(ctx, err, form) 197 176 return 198 177 } 199 178 200 179 if err := mirror_service.UpdateAddress(ctx.Repo.Mirror, address); err != nil { 201 180 ctx.ServerError("UpdateAddress", err) 181 181 + return 182 182 + } 183 183 + 184 184 + form.LFS = form.LFS && setting.LFS.StartServer 185 185 + 186 186 + if len(form.LFSEndpoint) > 0 { 187 187 + ep := lfs.DetermineEndpoint("", form.LFSEndpoint) 188 188 + if ep == nil { 189 189 + ctx.Data["Err_LFSEndpoint"] = true 190 190 + ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_lfs_endpoint"), tplSettingsOptions, &form) 191 191 + return 192 192 + } 193 193 + err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User) 194 194 + if err != nil { 195 195 + ctx.Data["Err_LFSEndpoint"] = true 196 196 + handleSettingRemoteAddrError(ctx, err, form) 197 197 + return 198 198 + } 199 199 + } 200 200 + 201 201 + ctx.Repo.Mirror.LFS = form.LFS 202 202 + ctx.Repo.Mirror.LFSEndpoint = form.LFSEndpoint 203 203 + if err := models.UpdateMirror(ctx.Repo.Mirror); err != nil { 204 204 + ctx.ServerError("UpdateMirror", err) 202 205 return 203 206 } 204 207 ··· 613 616 default: 614 617 ctx.NotFound("", nil) 615 618 } 619 619 + } 620 620 + 621 621 + func handleSettingRemoteAddrError(ctx *context.Context, err error, form *forms.RepoSettingForm) { 622 622 + if models.IsErrInvalidCloneAddr(err) { 623 623 + addrErr := err.(*models.ErrInvalidCloneAddr) 624 624 + switch { 625 625 + case addrErr.IsProtocolInvalid: 626 626 + ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tplSettingsOptions, form) 627 627 + case addrErr.IsURLError: 628 628 + ctx.RenderWithErr(ctx.Tr("form.url_error"), tplSettingsOptions, form) 629 629 + case addrErr.IsPermissionDenied: 630 630 + if addrErr.LocalPath { 631 631 + ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, form) 632 632 + } else if len(addrErr.PrivateNet) == 0 { 633 633 + ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, form) 634 634 + } else { 635 635 + ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tplSettingsOptions, form) 636 636 + } 637 637 + case addrErr.IsInvalidPath: 638 638 + ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, form) 639 639 + default: 640 640 + ctx.ServerError("Unknown error", err) 641 641 + } 642 642 + } 643 643 + ctx.RenderWithErr(ctx.Tr("repo.mirror_address_url_invalid"), tplSettingsOptions, form) 616 644 } 617 645 618 646 // Collaboration render a repository's collaboration page

+54 -55

routers/repo/view.go

reviewed

··· 274 274 275 275 // FIXME: what happens when README file is an image? 276 276 if isTextFile && setting.LFS.StartServer { 277 277 - meta := lfs.IsPointerFile(&buf) 278 278 - if meta != nil { 279 279 - meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid) 277 277 + pointer, _ := lfs.ReadPointerFromBuffer(buf) 278 278 + if pointer.IsValid() { 279 279 + meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) 280 280 if err != nil && err != models.ErrLFSObjectNotExist { 281 281 ctx.ServerError("GetLFSMetaObject", err) 282 282 return 283 283 } 284 284 - } 284 284 + if meta != nil { 285 285 + ctx.Data["IsLFSFile"] = true 286 286 + isLFSFile = true 287 287 + 288 288 + // OK read the lfs object 289 289 + var err error 290 290 + dataRc, err = lfs.ReadMetaObject(pointer) 291 291 + if err != nil { 292 292 + ctx.ServerError("ReadMetaObject", err) 293 293 + return 294 294 + } 295 295 + defer dataRc.Close() 285 296 286 286 - if meta != nil { 287 287 - ctx.Data["IsLFSFile"] = true 288 288 - isLFSFile = true 297 297 + buf = make([]byte, 1024) 298 298 + n, err = dataRc.Read(buf) 299 299 + if err != nil { 300 300 + ctx.ServerError("Data", err) 301 301 + return 302 302 + } 303 303 + buf = buf[:n] 289 304 290 290 - // OK read the lfs object 291 291 - var err error 292 292 - dataRc, err = lfs.ReadMetaObject(meta) 293 293 - if err != nil { 294 294 - ctx.ServerError("ReadMetaObject", err) 295 295 - return 296 296 - } 297 297 - defer dataRc.Close() 305 305 + isTextFile = base.IsTextFile(buf) 306 306 + ctx.Data["IsTextFile"] = isTextFile 298 307 299 299 - buf = make([]byte, 1024) 300 300 - n, err = dataRc.Read(buf) 301 301 - if err != nil { 302 302 - ctx.ServerError("Data", err) 303 303 - return 308 308 + fileSize = meta.Size 309 309 + ctx.Data["FileSize"] = meta.Size 310 310 + filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name)) 311 311 + ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, filenameBase64) 304 312 } 305 305 - buf = buf[:n] 306 306 - 307 307 - isTextFile = base.IsTextFile(buf) 308 308 - ctx.Data["IsTextFile"] = isTextFile 309 309 - 310 310 - fileSize = meta.Size 311 311 - ctx.Data["FileSize"] = meta.Size 312 312 - filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name)) 313 313 - ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, filenameBase64) 314 313 } 315 314 } 316 315 ··· 400 399 401 400 //Check for LFS meta file 402 401 if isTextFile && setting.LFS.StartServer { 403 403 - meta := lfs.IsPointerFile(&buf) 404 404 - if meta != nil { 405 405 - meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid) 402 402 + pointer, _ := lfs.ReadPointerFromBuffer(buf) 403 403 + if pointer.IsValid() { 404 404 + meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) 406 405 if err != nil && err != models.ErrLFSObjectNotExist { 407 406 ctx.ServerError("GetLFSMetaObject", err) 408 407 return 409 408 } 410 410 - } 411 411 - if meta != nil { 412 412 - isLFSFile = true 409 409 + if meta != nil { 410 410 + isLFSFile = true 413 411 414 414 - // OK read the lfs object 415 415 - var err error 416 416 - dataRc, err = lfs.ReadMetaObject(meta) 417 417 - if err != nil { 418 418 - ctx.ServerError("ReadMetaObject", err) 419 419 - return 420 420 - } 421 421 - defer dataRc.Close() 412 412 + // OK read the lfs object 413 413 + var err error 414 414 + dataRc, err = lfs.ReadMetaObject(pointer) 415 415 + if err != nil { 416 416 + ctx.ServerError("ReadMetaObject", err) 417 417 + return 418 418 + } 419 419 + defer dataRc.Close() 422 420 423 423 - buf = make([]byte, 1024) 424 424 - n, err = dataRc.Read(buf) 425 425 - // Error EOF don't mean there is an error, it just means we read to 426 426 - // the end 427 427 - if err != nil && err != io.EOF { 428 428 - ctx.ServerError("Data", err) 429 429 - return 430 430 - } 431 431 - buf = buf[:n] 421 421 + buf = make([]byte, 1024) 422 422 + n, err = dataRc.Read(buf) 423 423 + // Error EOF don't mean there is an error, it just means we read to 424 424 + // the end 425 425 + if err != nil && err != io.EOF { 426 426 + ctx.ServerError("Data", err) 427 427 + return 428 428 + } 429 429 + buf = buf[:n] 432 430 433 433 - isTextFile = base.IsTextFile(buf) 434 434 - fileSize = meta.Size 435 435 - ctx.Data["RawFileLink"] = fmt.Sprintf("%s/media/%s/%s", ctx.Repo.RepoLink, ctx.Repo.BranchNameSubURL(), ctx.Repo.TreePath) 431 431 + isTextFile = base.IsTextFile(buf) 432 432 + fileSize = meta.Size 433 433 + ctx.Data["RawFileLink"] = fmt.Sprintf("%s/media/%s/%s", ctx.Repo.RepoLink, ctx.Repo.BranchNameSubURL(), ctx.Repo.TreePath) 434 434 + } 436 435 } 437 436 } 438 437

+1 -1

routers/routes/web.go

reviewed

··· 16 16 "code.gitea.io/gitea/models" 17 17 "code.gitea.io/gitea/modules/context" 18 18 "code.gitea.io/gitea/modules/httpcache" 19 19 - "code.gitea.io/gitea/modules/lfs" 20 19 "code.gitea.io/gitea/modules/log" 21 20 "code.gitea.io/gitea/modules/metrics" 22 21 "code.gitea.io/gitea/modules/public" ··· 38 37 "code.gitea.io/gitea/routers/user" 39 38 userSetting "code.gitea.io/gitea/routers/user/setting" 40 39 "code.gitea.io/gitea/services/forms" 40 40 + "code.gitea.io/gitea/services/lfs" 41 41 "code.gitea.io/gitea/services/mailer" 42 42 43 43 // to registers all internal adapters

+4

services/forms/repo_form.go

reviewed

··· 71 71 // required: true 72 72 RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"` 73 73 Mirror bool `json:"mirror"` 74 74 + LFS bool `json:"lfs"` 75 75 + LFSEndpoint string `json:"lfs_endpoint"` 74 76 Private bool `json:"private"` 75 77 Description string `json:"description" binding:"MaxSize(255)"` 76 78 Wiki bool `json:"wiki"` ··· 118 120 MirrorAddress string 119 121 MirrorUsername string 120 122 MirrorPassword string 123 123 + LFS bool `form:"mirror_lfs"` 124 124 + LFSEndpoint string `form:"mirror_lfs_endpoint"` 121 125 Private bool 122 126 Template bool 123 127 EnablePrune bool

+5 -4

services/gitdiff/gitdiff.go

reviewed

··· 25 25 "code.gitea.io/gitea/modules/charset" 26 26 "code.gitea.io/gitea/modules/git" 27 27 "code.gitea.io/gitea/modules/highlight" 28 28 + "code.gitea.io/gitea/modules/lfs" 28 29 "code.gitea.io/gitea/modules/log" 29 30 "code.gitea.io/gitea/modules/process" 30 31 "code.gitea.io/gitea/modules/setting" ··· 1077 1078 curSection.Lines[len(curSection.Lines)-1].Content = line 1078 1079 1079 1080 // handle LFS 1080 1080 - if line[1:] == models.LFSMetaFileIdentifier { 1081 1081 + if line[1:] == lfs.MetaFileIdentifier { 1081 1082 curFileLFSPrefix = true 1082 1082 - } else if curFileLFSPrefix && strings.HasPrefix(line[1:], models.LFSMetaFileOidPrefix) { 1083 1083 - oid := strings.TrimPrefix(line[1:], models.LFSMetaFileOidPrefix) 1083 1083 + } else if curFileLFSPrefix && strings.HasPrefix(line[1:], lfs.MetaFileOidPrefix) { 1084 1084 + oid := strings.TrimPrefix(line[1:], lfs.MetaFileOidPrefix) 1084 1085 if len(oid) == 64 { 1085 1085 - m := &models.LFSMetaObject{Oid: oid} 1086 1086 + m := &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}} 1086 1087 count, err := models.Count(m) 1087 1088 1088 1089 if err == nil && count > 0 {

+15 -6

services/mirror/mirror.go

reviewed

··· 16 16 "code.gitea.io/gitea/modules/cache" 17 17 "code.gitea.io/gitea/modules/git" 18 18 "code.gitea.io/gitea/modules/graceful" 19 19 + "code.gitea.io/gitea/modules/lfs" 19 20 "code.gitea.io/gitea/modules/log" 20 21 "code.gitea.io/gitea/modules/notification" 21 22 repo_module "code.gitea.io/gitea/modules/repository" ··· 206 207 } 207 208 208 209 // runSync returns true if sync finished without error. 209 209 - func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { 210 210 + func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) { 210 211 repoPath := m.Repo.RepoPath() 211 212 wikiPath := m.Repo.WikiPath() 212 213 timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second ··· 253 254 log.Error("OpenRepository: %v", err) 254 255 return nil, false 255 256 } 257 257 + defer gitRepo.Close() 256 258 257 259 log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) 258 260 if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { 259 259 - gitRepo.Close() 260 261 log.Error("Failed to synchronize tags to releases for repository: %v", err) 261 262 } 262 262 - gitRepo.Close() 263 263 + 264 264 + if m.LFS && setting.LFS.StartServer { 265 265 + log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) 266 266 + readAddress(m) 267 267 + ep := lfs.DetermineEndpoint(m.Address, m.LFSEndpoint) 268 268 + if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil { 269 269 + log.Error("Failed to synchronize LFS objects for repository: %v", err) 270 270 + } 271 271 + } 263 272 264 273 log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) 265 274 if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil { ··· 378 387 mirrorQueue.Close() 379 388 return 380 389 case repoID := <-mirrorQueue.Queue(): 381 381 - syncMirror(repoID) 390 390 + syncMirror(ctx, repoID) 382 391 } 383 392 } 384 393 } 385 394 386 386 - func syncMirror(repoID string) { 395 395 + func syncMirror(ctx context.Context, repoID string) { 387 396 log.Trace("SyncMirrors [repo_id: %v]", repoID) 388 397 defer func() { 389 398 err := recover() ··· 403 412 } 404 413 405 414 log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo) 406 406 - results, ok := runSync(m) 415 415 + results, ok := runSync(ctx, m) 407 416 if !ok { 408 417 return 409 418 }

+5 -3

services/mirror/mirror_test.go

reviewed

··· 48 48 }) 49 49 assert.NoError(t, err) 50 50 51 51 - mirror, err := repository.MigrateRepositoryGitData(context.Background(), user, mirrorRepo, opts) 51 51 + ctx := context.Background() 52 52 + 53 53 + mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts) 52 54 assert.NoError(t, err) 53 55 54 56 gitRepo, err := git.OpenRepository(repoPath) ··· 74 76 err = mirror.GetMirror() 75 77 assert.NoError(t, err) 76 78 77 77 - _, ok := runSync(mirror.Mirror) 79 79 + _, ok := runSync(ctx, mirror.Mirror) 78 80 assert.True(t, ok) 79 81 80 82 count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions) ··· 85 87 assert.NoError(t, err) 86 88 assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true)) 87 89 88 88 - _, ok = runSync(mirror.Mirror) 90 90 + _, ok = runSync(ctx, mirror.Mirror) 89 91 assert.True(t, ok) 90 92 91 93 count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions)

+13 -4

services/pull/lfs.go

reviewed

··· 70 70 defer wg.Done() 71 71 defer catFileBatchReader.Close() 72 72 73 73 + contentStore := lfs.NewContentStore() 74 74 + 73 75 bufferedReader := bufio.NewReader(catFileBatchReader) 74 76 buf := make([]byte, 1025) 75 77 for { ··· 101 103 } 102 104 pointerBuf = pointerBuf[:size] 103 105 // Now we need to check if the pointerBuf is an LFS pointer 104 104 - pointer := lfs.IsPointerFile(&pointerBuf) 105 105 - if pointer == nil { 106 106 + pointer, _ := lfs.ReadPointerFromBuffer(pointerBuf) 107 107 + if !pointer.IsValid() { 108 108 + continue 109 109 + } 110 110 + 111 111 + exist, _ := contentStore.Exists(pointer) 112 112 + if !exist { 106 113 continue 107 114 } 115 115 + 108 116 // Then we need to check that this pointer is in the db 109 117 if _, err := pr.HeadRepo.GetLFSMetaObjectByOid(pointer.Oid); err != nil { 110 118 if err == models.ErrLFSObjectNotExist { ··· 117 125 // OK we have a pointer that is associated with the head repo 118 126 // and is actually a file in the LFS 119 127 // Therefore it should be associated with the base repo 120 120 - pointer.RepositoryID = pr.BaseRepoID 121 121 - if _, err := models.NewLFSMetaObject(pointer); err != nil { 128 128 + meta := &models.LFSMetaObject{Pointer: pointer} 129 129 + meta.RepositoryID = pr.BaseRepoID 130 130 + if _, err := models.NewLFSMetaObject(meta); err != nil { 122 131 _ = catFileBatchReader.CloseWithError(err) 123 132 break 124 133 }

+1 -13

templates/repo/migrate/git.tmpl

reviewed

··· 15 15 <input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required> 16 16 <span class="help"> 17 17 {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} 18 18 - {{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} 19 18 </span> 20 19 </div> 21 20 <div class="inline field {{if .Err_Auth}}error{{end}}"> ··· 28 27 <input id="auth_password" name="auth_password" type="password" value="{{.auth_password}}"> 29 28 </div> 30 29 31 31 - <div class="inline field"> 32 32 - <label>{{.i18n.Tr "repo.migrate_options"}}</label> 33 33 - <div class="ui checkbox"> 34 34 - {{if .DisableMirrors}} 35 35 - <input id="mirror" name="mirror" type="checkbox" readonly> 36 36 - <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label> 37 37 - {{else}} 38 38 - <input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}> 39 39 - <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label> 40 40 - {{end}} 41 41 - </div> 42 42 - </div> 30 30 + {{template "repo/migrate/options" .}} 43 31 44 32 <div class="ui divider"></div> 45 33

+2 -14

templates/repo/migrate/gitea.tmpl

reviewed

··· 15 15 <input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required> 16 16 <span class="help"> 17 17 {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} 18 18 - {{if .LFSActive}}<br />{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} 19 18 </span> 20 19 </div> 21 20 22 21 <div class="inline field {{if .Err_Auth}}error{{end}}"> 23 22 <label for="auth_token">{{.i18n.Tr "access_token"}}</label> 24 23 <input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}} data-need-clear="true" {{end}}> 25 25 - <a target=”_blank” href="https://docs.gitea.io/en-us/api-usage">{{svg "octicon-question"}}</a> 24 24 + <a target="_blank" href="https://docs.gitea.io/en-us/api-usage">{{svg "octicon-question"}}</a> 26 25 </div> 27 26 28 28 - <div class="inline field"> 29 29 - <label>{{.i18n.Tr "repo.migrate_options"}}</label> 30 30 - <div class="ui checkbox"> 31 31 - {{if .DisableMirrors}} 32 32 - <input id="mirror" name="mirror" type="checkbox" readonly> 33 33 - <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label> 34 34 - {{else}} 35 35 - <input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}> 36 36 - <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label> 37 37 - {{end}} 38 38 - </div> 39 39 - </div> 27 27 + {{template "repo/migrate/options" .}} 40 28 41 29 <span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span> 42 30 <div id="migrate_items">

+2 -14

templates/repo/migrate/github.tmpl

reviewed

··· 15 15 <input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required> 16 16 <span class="help"> 17 17 {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} 18 18 - {{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} 19 18 </span> 20 19 </div> 21 20 22 21 <div class="inline field {{if .Err_Auth}}error{{end}}"> 23 22 <label for="auth_token">{{.i18n.Tr "access_token"}}</label> 24 23 <input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}}data-need-clear="true"{{end}}> 25 25 - <a target=”_blank” href="https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token">{{svg "octicon-question"}}</a> 24 24 + <a target="_blank" href="https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token">{{svg "octicon-question"}}</a> 26 25 </div> 27 26 28 28 - <div class="inline field"> 29 29 - <label>{{.i18n.Tr "repo.migrate_options"}}</label> 30 30 - <div class="ui checkbox"> 31 31 - {{if .DisableMirrors}} 32 32 - <input id="mirror" name="mirror" type="checkbox" readonly> 33 33 - <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label> 34 34 - {{else}} 35 35 - <input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}> 36 36 - <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label> 37 37 - {{end}} 38 38 - </div> 39 39 - </div> 27 27 + {{template "repo/migrate/options" .}} 40 28 41 29 <span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span> 42 30 <div id="migrate_items">

+2 -14

templates/repo/migrate/gitlab.tmpl

reviewed

··· 15 15 <input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required> 16 16 <span class="help"> 17 17 {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} 18 18 - {{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} 19 18 </span> 20 19 </div> 21 20 22 21 <div class="inline field {{if .Err_Auth}}error{{end}}"> 23 22 <label for="auth_token">{{.i18n.Tr "access_token"}}</label> 24 23 <input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}}data-need-clear="true"{{end}}> 25 25 - <a target=”_blank” href="https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html">{{svg "octicon-question"}}</a> 24 24 + <a target="_blank" href="https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html">{{svg "octicon-question"}}</a> 26 25 </div> 27 26 28 28 - <div class="inline field"> 29 29 - <label>{{.i18n.Tr "repo.migrate_options"}}</label> 30 30 - <div class="ui checkbox"> 31 31 - {{if .DisableMirrors}} 32 32 - <input id="mirror" name="mirror" type="checkbox" readonly> 33 33 - <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label> 34 34 - {{else}} 35 35 - <input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}> 36 36 - <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label> 37 37 - {{end}} 38 38 - </div> 39 39 - </div> 27 27 + {{template "repo/migrate/options" .}} 40 28 41 29 <span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span> 42 30 <div id="migrate_items">

+2 -14

templates/repo/migrate/gogs.tmpl

reviewed

··· 15 15 <input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required> 16 16 <span class="help"> 17 17 {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} 18 18 - {{if .LFSActive}}<br />{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}} 19 18 </span> 20 19 </div> 21 20 22 21 <div class="inline field {{if .Err_Auth}}error{{end}}"> 23 22 <label for="auth_token">{{.i18n.Tr "access_token"}}</label> 24 23 <input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}} data-need-clear="true" {{end}}> 25 25 - <!-- <a target=”_blank” href="https://docs.gitea.io/en-us/api-usage">{{svg "octicon-question"}}</a> --> 24 24 + <!-- <a target="_blank" href="https://docs.gitea.io/en-us/api-usage">{{svg "octicon-question"}}</a> --> 26 25 </div> 27 26 28 28 - <div class="inline field"> 29 29 - <label>{{.i18n.Tr "repo.migrate_options"}}</label> 30 30 - <div class="ui checkbox"> 31 31 - {{if .DisableMirrors}} 32 32 - <input id="mirror" name="mirror" type="checkbox" readonly> 33 33 - <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label> 34 34 - {{else}} 35 35 - <input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}> 36 36 - <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label> 37 37 - {{end}} 38 38 - </div> 39 39 - </div> 27 27 + {{template "repo/migrate/options" .}} 40 28 41 29 <span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span> 42 30 <div id="migrate_items">

+29

templates/repo/migrate/options.tmpl

reviewed

··· 1 1 + <div class="inline field"> 2 2 + <label>{{.i18n.Tr "repo.migrate_options"}}</label> 3 3 + <div class="ui checkbox"> 4 4 + {{if .DisableMirrors}} 5 5 + <input id="mirror" name="mirror" type="checkbox" readonly> 6 6 + <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label> 7 7 + {{else}} 8 8 + <input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}> 9 9 + <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label> 10 10 + {{end}} 11 11 + </div> 12 12 + </div> 13 13 + {{if .LFSActive}} 14 14 + <div class="inline field"> 15 15 + <label></label> 16 16 + <div class="ui checkbox"> 17 17 + <input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}> 18 18 + <label>{{.i18n.Tr "repo.migrate_options_lfs"}}</label> 19 19 + </div> 20 20 + <span id="lfs_settings" style="display:none">(<a id="lfs_settings_show" href="#">{{.i18n.Tr "repo.settings.advanced_settings"}}</a>)</span> 21 21 + </div> 22 22 + <div id="lfs_endpoint" style="display:none"> 23 23 + <span class="help">{{.i18n.Tr "repo.migrate_options_lfs_endpoint.description" "https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md#server-discovery" | Str2html}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate_options_lfs_endpoint.description.local"}}{{end}}</span> 24 24 + <div class="inline field {{if .Err_LFSEndpoint}}error{{end}}"> 25 25 + <label>{{.i18n.Tr "repo.migrate_options_lfs_endpoint.label"}}</label> 26 26 + <input name="lfs_endpoint" value="{{.lfs_endpoint}}" placeholder="{{.i18n.Tr "repo.migrate_options_lfs_endpoint.placeholder"}}"> 27 27 + </div> 28 28 + </div> 29 29 + {{end}}

+17 -2

templates/repo/settings/options.tmpl

reviewed

··· 81 81 <div class="inline field {{if .Err_EnablePrune}}error{{end}}"> 82 82 <label>{{.i18n.Tr "repo.mirror_prune"}}</label> 83 83 <div class="ui checkbox"> 84 84 - <input id="enable_prune" name="enable_prune" type="checkbox" {{if .MirrorEnablePrune}}checked{{end}}> 85 85 - <label>{{.i18n.Tr "repo.mirror_prune_desc"}}</label> 84 84 + <input id="enable_prune" name="enable_prune" type="checkbox" {{if .MirrorEnablePrune}}checked{{end}}> 85 85 + <label>{{.i18n.Tr "repo.mirror_prune_desc"}}</label> 86 86 </div> 87 87 </div> 88 88 <div class="inline field {{if .Err_Interval}}error{{end}}"> ··· 111 111 </div> 112 112 </div> 113 113 </div> 114 114 + 115 115 + {{if .LFSStartServer}} 116 116 + <div class="inline field"> 117 117 + <label>{{.i18n.Tr "repo.mirror_lfs"}}</label> 118 118 + <div class="ui checkbox"> 119 119 + <input id="mirror_lfs" name="mirror_lfs" type="checkbox" {{if .Mirror.LFS}}checked{{end}}> 120 120 + <label>{{.i18n.Tr "repo.mirror_lfs_desc"}}</label> 121 121 + </div> 122 122 + </div> 123 123 + <div class="field {{if .Err_LFSEndpoint}}error{{end}}"> 124 124 + <label for="mirror_lfs_endpoint">{{.i18n.Tr "repo.mirror_lfs_endpoint"}}</label> 125 125 + <input id="mirror_lfs_endpoint" name="mirror_lfs_endpoint" value="{{.Mirror.LFSEndpoint}}" placeholder="{{.i18n.Tr "repo.migrate_options_lfs_endpoint.placeholder"}}"> 126 126 + <p class="help">{{.i18n.Tr "repo.mirror_lfs_endpoint_desc" "https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md#server-discovery" | Str2html}}</p> 127 127 + </div> 128 128 + {{end}} 114 129 115 130 <div class="field"> 116 131 <button class="ui green button">{{$.i18n.Tr "repo.settings.update_settings"}}</button>

+16

templates/swagger/v1_json.tmpl

reviewed

··· 14669 14669 "type": "boolean", 14670 14670 "x-go-name": "Labels" 14671 14671 }, 14672 14672 + "lfs": { 14673 14673 + "type": "boolean", 14674 14674 + "x-go-name": "LFS" 14675 14675 + }, 14676 14676 + "lfs_endpoint": { 14677 14677 + "type": "string", 14678 14678 + "x-go-name": "LFSEndpoint" 14679 14679 + }, 14672 14680 "milestones": { 14673 14681 "type": "boolean", 14674 14682 "x-go-name": "Milestones" ··· 14747 14755 "labels": { 14748 14756 "type": "boolean", 14749 14757 "x-go-name": "Labels" 14758 14758 + }, 14759 14759 + "lfs": { 14760 14760 + "type": "boolean", 14761 14761 + "x-go-name": "LFS" 14762 14762 + }, 14763 14763 + "lfs_endpoint": { 14764 14764 + "type": "string", 14765 14765 + "x-go-name": "LFSEndpoint" 14750 14766 }, 14751 14767 "milestones": { 14752 14768 "type": "boolean",

+12

web_src/js/features/migration.js

reviewed

··· 3 3 const $pass = $('#auth_password'); 4 4 const $token = $('#auth_token'); 5 5 const $mirror = $('#mirror'); 6 6 + const $lfs = $('#lfs'); 7 7 + const $lfsSettings = $('#lfs_settings'); 8 8 + const $lfsEndpoint = $('#lfs_endpoint'); 6 9 const $items = $('#migrate_items').find('input[type=checkbox]'); 7 10 8 11 export default function initMigration() { 9 12 checkAuth(); 13 13 + setLFSSettingsVisibility(); 10 14 11 15 $user.on('keyup', () => {checkItems(false)}); 12 16 $pass.on('keyup', () => {checkItems(false)}); 13 17 $token.on('keyup', () => {checkItems(true)}); 14 18 $mirror.on('change', () => {checkItems(true)}); 19 19 + $('#lfs_settings_show').on('click', () => { $lfsEndpoint.show(); return false }); 20 20 + $lfs.on('change', setLFSSettingsVisibility); 15 21 16 22 const $cloneAddr = $('#clone_addr'); 17 23 $cloneAddr.on('change', () => { ··· 46 52 $items.attr('disabled', true); 47 53 } 48 54 } 55 55 + 56 56 + function setLFSSettingsVisibility() { 57 57 + const visible = $lfs.is(':checked'); 58 58 + $lfsSettings.toggle(visible); 59 59 + $lfsEndpoint.hide(); 60 60 + }