Fast implementation of Git in pure Go codeberg.org/lindenii/furgit
git go
6
fork

Configure Feed

Select the types of activity you want to include in your feed.

network/receivepack: Use dual

Runxi Yu da621b97 84342973

+216 -740
+52 -1
cmd/receivepack9418/conn.go
··· 6 6 "fmt" 7 7 "log" 8 8 "net" 9 + "os" 9 10 "strings" 10 11 11 12 "codeberg.org/lindenii/furgit/network/receivepack" 13 + objectdual "codeberg.org/lindenii/furgit/object/store/dual" 14 + objectloose "codeberg.org/lindenii/furgit/object/store/loose" 15 + objectpacked "codeberg.org/lindenii/furgit/object/store/packed" 12 16 ) 13 17 14 18 func (srv *server) handleConn(conn net.Conn) { ··· 38 42 39 43 gitProtocol := strings.Join(req.ExtraParameters, ":") 40 44 45 + objectIngress, cleanupObjectIngress, err := srv.openObjectIngress() 46 + if err != nil { 47 + writeErrPkt(writer, fmt.Sprintf("object ingress unavailable: %v", err)) 48 + _ = writer.Flush() 49 + 50 + log.Printf("receivepack9418: %s: object ingress unavailable: %v", conn.RemoteAddr(), err) 51 + 52 + return 53 + } 54 + 55 + defer cleanupObjectIngress() 56 + 41 57 opts := receivepack.Options{ 42 58 GitProtocol: gitProtocol, 43 59 Algorithm: srv.repo.Algorithm(), 44 60 Refs: srv.repo.Refs(), 45 61 ExistingObjects: srv.repo.Objects(), 46 - ObjectsRoot: srv.objectsRoot, 62 + ObjectIngress: objectIngress, 47 63 } 48 64 49 65 err = receivepack.ReceivePack(context.Background(), writer, reader, opts) ··· 69 85 return 70 86 } 71 87 } 88 + 89 + func (srv *server) openObjectIngress() (*objectdual.Dual, func(), error) { 90 + err := srv.objectsRoot.Mkdir("pack", 0o755) 91 + if err != nil && !os.IsExist(err) { 92 + return nil, nil, err 93 + } 94 + 95 + packRoot, err := srv.objectsRoot.OpenRoot("pack") 96 + if err != nil { 97 + return nil, nil, err 98 + } 99 + 100 + looseStore, err := objectloose.New(srv.objectsRoot, srv.repo.Algorithm()) 101 + if err != nil { 102 + _ = packRoot.Close() 103 + 104 + return nil, nil, err 105 + } 106 + 107 + packedStore, err := objectpacked.New(packRoot, srv.repo.Algorithm(), objectpacked.Options{WriteRev: true}) 108 + if err != nil { 109 + _ = looseStore.Close() 110 + _ = packRoot.Close() 111 + 112 + return nil, nil, err 113 + } 114 + 115 + cleanup := func() { 116 + _ = packedStore.Close() 117 + _ = looseStore.Close() 118 + _ = packRoot.Close() 119 + } 120 + 121 + return objectdual.New(looseStore, packedStore), cleanup, nil 122 + }
-2
network/receivepack/hooks/reject_force_push.go
··· 23 23 24 24 objects := objectmix.New(req.QuarantinedObjects, req.ExistingObjects) 25 25 26 - defer func() { _ = objects.Close() }() 27 - 28 26 queries := commitquery.New(fetch.New(objects), req.CommitGraph) 29 27 30 28 decisions := make([]receivepack.UpdateDecision, len(req.Updates))
+62 -14
network/receivepack/int_test.go
··· 15 15 receivepack "codeberg.org/lindenii/furgit/network/receivepack" 16 16 receivepackhooks "codeberg.org/lindenii/furgit/network/receivepack/hooks" 17 17 objectid "codeberg.org/lindenii/furgit/object/id" 18 + objectstore "codeberg.org/lindenii/furgit/object/store" 19 + objectdual "codeberg.org/lindenii/furgit/object/store/dual" 20 + objectloose "codeberg.org/lindenii/furgit/object/store/loose" 21 + objectpacked "codeberg.org/lindenii/furgit/object/store/packed" 18 22 ) 19 23 20 24 func TestReceivePackDeleteOnlyAtomicDeleteSucceeds(t *testing.T) { ··· 301 305 }) 302 306 } 303 307 304 - func TestReceivePackPackRequestWithoutObjectsRootReportsNotConfigured(t *testing.T) { 308 + func TestReceivePackPackRequestWithoutObjectIngressReportsNotConfigured(t *testing.T) { 305 309 t.Parallel() 306 310 307 311 //nolint:thelper ··· 334 338 } 335 339 336 340 got := output.String() 337 - if !strings.Contains(got, "unpack objects root not configured\n") { 341 + if !strings.Contains(got, "unpack object ingress not configured\n") { 338 342 t.Fatalf("unexpected receive-pack output %q", got) 339 343 } 340 344 }) ··· 352 356 353 357 receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) 354 358 repo := receiver.OpenRepository(t) 355 - objectsRoot := receiver.OpenObjectsRoot(t) 359 + objectIngress := openReceivePackIngress(t, receiver, algo) 356 360 357 361 packStream := sender.PackObjectsReader(t, []string{commitID.String()}, false) 358 362 t.Cleanup(func() { ··· 377 381 Algorithm: algo, 378 382 Refs: repo.Refs(), 379 383 ExistingObjects: repo.Objects(), 380 - ObjectsRoot: objectsRoot, 384 + ObjectIngress: objectIngress, 381 385 }, 382 386 ) 383 387 if err != nil { ··· 423 427 424 428 receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) 425 429 repo := receiver.OpenRepository(t) 426 - objectsRoot := receiver.OpenObjectsRoot(t) 430 + objectIngress := openReceivePackIngress(t, receiver, algo) 427 431 428 432 packStream := sender.PackObjectsReader(t, []string{commitID.String()}, false) 429 433 t.Cleanup(func() { ··· 449 453 Algorithm: algo, 450 454 Refs: repo.Refs(), 451 455 ExistingObjects: repo.Objects(), 452 - ObjectsRoot: objectsRoot, 456 + ObjectIngress: objectIngress, 453 457 Hook: func(ctx context.Context, req receivepack.HookRequest) ([]receivepack.UpdateDecision, error) { 454 458 hookCalled = true 455 459 ··· 658 662 testRepo.UpdateRef(t, "refs/heads/main", currentID) 659 663 660 664 repo := testRepo.OpenRepository(t) 661 - objectsRoot := testRepo.OpenObjectsRoot(t) 665 + objectIngress := openReceivePackIngress(t, testRepo, algo) 662 666 packStream := testRepo.PackObjectsReader(t, []string{forcedID.String(), "^" + currentID.String()}, false) 663 667 t.Cleanup(func() { 664 668 _ = packStream.Close() ··· 682 686 Algorithm: algo, 683 687 Refs: repo.Refs(), 684 688 ExistingObjects: repo.Objects(), 685 - ObjectsRoot: objectsRoot, 689 + ObjectIngress: objectIngress, 686 690 Hook: receivepackhooks.RejectForcePush(), 687 691 }, 688 692 ) ··· 765 769 766 770 receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) 767 771 repo := receiver.OpenRepository(t) 768 - objectsRoot := receiver.OpenObjectsRoot(t) 772 + objectIngress := openReceivePackIngress(t, receiver, algo) 769 773 770 774 stdout, stderr, clientErr, serverErr := runGitPushFD( 771 775 t, ··· 774 778 Algorithm: algo, 775 779 Refs: repo.Refs(), 776 780 ExistingObjects: repo.Objects(), 777 - ObjectsRoot: objectsRoot, 781 + ObjectIngress: objectIngress, 778 782 }, 779 783 "push", "--porcelain", "fd::3,4/test", "refs/heads/main:refs/heads/main", 780 784 ) ··· 815 819 receiver.UpdateRef(t, "refs/heads/main", commitID) 816 820 817 821 repo := receiver.OpenRepository(t) 818 - objectsRoot := receiver.OpenObjectsRoot(t) 822 + objectIngress := openReceivePackIngress(t, receiver, algo) 819 823 820 824 stdout, stderr, clientErr, serverErr := runGitPushFD( 821 825 t, ··· 824 828 Algorithm: algo, 825 829 Refs: repo.Refs(), 826 830 ExistingObjects: repo.Objects(), 827 - ObjectsRoot: objectsRoot, 831 + ObjectIngress: objectIngress, 828 832 }, 829 833 "push", "--porcelain", "fd::3,4/test", "refs/heads/main:refs/heads/topic", 830 834 ) ··· 911 915 receiver.UpdateRef(t, "refs/heads/main", currentID) 912 916 913 917 repo := receiver.OpenRepository(t) 914 - objectsRoot := receiver.OpenObjectsRoot(t) 918 + objectIngress := openReceivePackIngress(t, receiver, algo) 915 919 916 920 stdout, stderr, clientErr, serverErr := runGitPushFD( 917 921 t, ··· 920 924 Algorithm: algo, 921 925 Refs: repo.Refs(), 922 926 ExistingObjects: repo.Objects(), 923 - ObjectsRoot: objectsRoot, 927 + ObjectIngress: objectIngress, 924 928 Hook: receivepackhooks.RejectForcePush(), 925 929 }, 926 930 "push", "--porcelain", "--force", "fd::3,4/test", "refs/heads/main:refs/heads/main", ··· 958 962 959 963 func pktlineData(payload string) string { 960 964 return fmt.Sprintf("%04x%s", len(payload)+4, payload) 965 + } 966 + 967 + func openReceivePackIngress( 968 + tb testing.TB, 969 + testRepo *testgit.TestRepo, 970 + algo objectid.Algorithm, 971 + ) objectstore.Quarantiner { 972 + tb.Helper() 973 + 974 + objectsRoot := testRepo.OpenObjectsRoot(tb) 975 + 976 + err := objectsRoot.Mkdir("pack", 0o755) 977 + if err != nil && !os.IsExist(err) { 978 + tb.Fatalf("Mkdir(pack): %v", err) 979 + } 980 + 981 + packRoot, err := objectsRoot.OpenRoot("pack") 982 + if err != nil { 983 + tb.Fatalf("OpenRoot(pack): %v", err) 984 + } 985 + 986 + tb.Cleanup(func() { 987 + _ = packRoot.Close() 988 + }) 989 + 990 + looseStore, err := objectloose.New(objectsRoot, algo) 991 + if err != nil { 992 + tb.Fatalf("loose.New: %v", err) 993 + } 994 + 995 + tb.Cleanup(func() { 996 + _ = looseStore.Close() 997 + }) 998 + 999 + packedStore, err := objectpacked.New(packRoot, algo, objectpacked.Options{WriteRev: true}) 1000 + if err != nil { 1001 + tb.Fatalf("packed.New: %v", err) 1002 + } 1003 + 1004 + tb.Cleanup(func() { 1005 + _ = packedStore.Close() 1006 + }) 1007 + 1008 + return objectdual.New(looseStore, packedStore) 961 1009 } 962 1010 963 1011 type fileWriteFlusher struct {
+4 -9
network/receivepack/options.go
··· 1 1 package receivepack 2 2 3 3 import ( 4 - "os" 5 - 6 4 commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" 7 5 objectid "codeberg.org/lindenii/furgit/object/id" 8 6 objectstore "codeberg.org/lindenii/furgit/object/store" ··· 14 12 // ReceivePack borrows all configured dependencies. 15 13 // 16 14 // Refs and ExistingObjects are required and must be non-nil. 17 - // ObjectsRoot is required if the invocation may need to ingest or promote a 15 + // ObjectIngress is required if the invocation may need to ingest or quarantine a 18 16 // pack. 19 17 type Options struct { 20 18 // GitProtocol is the raw Git protocol version string from the transport, ··· 31 29 // ExistingObjects is the object store visible to the push before any newly 32 30 // uploaded quarantined objects are promoted. 33 31 ExistingObjects objectstore.Reader 32 + // ObjectIngress creates coordinated quarantines for quarantined object and 33 + // pack ingestion during the push. 34 + ObjectIngress objectstore.Quarantiner 34 35 // CommitGraph is an optional commit-graph snapshot corresponding to 35 36 // ExistingObjects. 36 37 CommitGraph *commitgraphread.Reader 37 - // ObjectsRoot is the permanent object storage root beneath which per-push 38 - // quarantine directories are derived. 39 - ObjectsRoot *os.Root 40 - // PromotedObjectPermissions, when non-nil, is applied to objects and 41 - // directories moved from quarantine into the permanent object store. 42 - PromotedObjectPermissions *PromotedObjectPermissions 43 38 // Hook, when non-nil, runs after pack ingestion into quarantine and before 44 39 // quarantine promotion or ref updates. Hook is borrowed for the duration of 45 40 // ReceivePack.
-27
network/receivepack/permissions.go
··· 1 - package receivepack 2 - 3 - import ( 4 - "io/fs" 5 - 6 - "codeberg.org/lindenii/furgit/network/receivepack/service" 7 - ) 8 - 9 - // PromotedObjectPermissions configures the destination permissions applied to 10 - // objects and directories promoted out of quarantine. 11 - type PromotedObjectPermissions struct { 12 - DirMode fs.FileMode 13 - FileMode fs.FileMode 14 - } 15 - 16 - func translatePromotedObjectPermissions( 17 - perms *PromotedObjectPermissions, 18 - ) *service.PromotedObjectPermissions { 19 - if perms == nil { 20 - return nil 21 - } 22 - 23 - return &service.PromotedObjectPermissions{ 24 - DirMode: perms.DirMode, 25 - FileMode: perms.FileMode, 26 - } 27 - }
+2 -5
network/receivepack/receivepack.go
··· 108 108 Algorithm: opts.Algorithm, 109 109 Refs: opts.Refs, 110 110 ExistingObjects: opts.ExistingObjects, 111 + ObjectIngress: opts.ObjectIngress, 111 112 CommitGraph: opts.CommitGraph, 112 - ObjectsRoot: opts.ObjectsRoot, 113 113 Progress: progress, 114 - PromotedObjectPermissions: translatePromotedObjectPermissions( 115 - opts.PromotedObjectPermissions, 116 - ), 117 - Hook: translateHook(opts.Hook), 114 + Hook: translateHook(opts.Hook), 118 115 HookIO: service.HookIO{ 119 116 Progress: progress, 120 117 Error: protoSession.ErrorWriter(),
+11 -16
network/receivepack/service/execute.go
··· 2 2 3 3 import ( 4 4 "context" 5 - "os" 6 5 7 6 "codeberg.org/lindenii/furgit/internal/utils" 7 + objectstore "codeberg.org/lindenii/furgit/object/store" 8 8 ) 9 9 10 10 // Execute validates one receive-pack request, optionally ingests its pack into ··· 15 15 result := &Result{ 16 16 Commands: make([]CommandResult, 0, len(req.Commands)), 17 17 } 18 - 19 - var ( 20 - quarantineName string 21 - quarantineRoot *os.Root 22 - err error 23 - ) 18 + var err error 24 19 25 - quarantineName, quarantineRoot, ok := service.ingestQuarantine(result, req.Commands, req) 20 + quarantine, ok := service.ingestQuarantine(result, req.Commands, req) 26 21 if !ok { 27 22 return result, nil 28 23 } 29 24 30 - if quarantineRoot != nil { 31 - defer func() { 32 - _ = quarantineRoot.Close() 33 - _ = service.opts.ObjectsRoot.RemoveAll(quarantineName) 34 - }() 25 + if quarantine != nil { 26 + defer func(q objectstore.Quarantine) { 27 + _ = q.Discard() 28 + }(quarantine) 35 29 } 36 30 37 31 for _, command := range req.Commands { ··· 51 45 ctx, 52 46 req, 53 47 req.Commands, 54 - quarantineName, 48 + quarantine, 55 49 ) 56 50 if !ok { 57 51 fillCommandErrors(result, req.Commands, errText) ··· 79 73 return result, nil 80 74 } 81 75 82 - if req.PackExpected && quarantineRoot != nil { 76 + if req.PackExpected && quarantine != nil { 83 77 // Git migrates quarantined objects into permanent storage immediately 84 78 // before starting ref updates. 85 79 utils.BestEffortFprintf(service.opts.Progress, "promoting quarantine...\r") 86 80 87 - err = service.promoteQuarantine(quarantineName, quarantineRoot) 81 + err := quarantine.Promote() 88 82 if err != nil { 89 83 utils.BestEffortFprintf(service.opts.Progress, "promoting quarantine: failed: %v.\n", err) 90 84 ··· 94 88 return result, nil 95 89 } 96 90 91 + quarantine = nil 97 92 utils.BestEffortFprintf(service.opts.Progress, "promoting quarantine: done.\n") 98 93 } 99 94
+21 -83
network/receivepack/service/ingest_quarantine.go
··· 1 1 package service 2 2 3 3 import ( 4 - "os" 5 - 6 - "codeberg.org/lindenii/furgit/format/packfile/ingest" 7 4 "codeberg.org/lindenii/furgit/internal/utils" 5 + objectstore "codeberg.org/lindenii/furgit/object/store" 8 6 ) 9 7 10 8 func (service *Service) ingestQuarantine( 11 9 result *Result, 12 10 commands []Command, 13 11 req *Request, 14 - ) (string, *os.Root, bool) { 12 + ) (objectstore.Quarantine, bool) { 15 13 if !req.PackExpected { 16 - return "", nil, true 14 + return nil, true 17 15 } 18 16 19 17 if req.Pack == nil { ··· 22 20 result.UnpackError = "missing pack stream" 23 21 fillCommandErrors(result, commands, "missing pack stream") 24 22 25 - return "", nil, false 23 + return nil, false 26 24 } 27 25 28 - if service.opts.ObjectsRoot == nil { 29 - utils.BestEffortFprintf(service.opts.Progress, "unpack failed: objects root not configured.\n") 26 + if service.opts.ObjectIngress == nil { 27 + utils.BestEffortFprintf(service.opts.Progress, "unpack failed: object ingress not configured.\n") 30 28 31 - result.UnpackError = "objects root not configured" 32 - fillCommandErrors(result, commands, "objects root not configured") 29 + result.UnpackError = "object ingress not configured" 30 + fillCommandErrors(result, commands, "object ingress not configured") 33 31 34 - return "", nil, false 32 + return nil, false 35 33 } 36 34 37 35 var err error ··· 43 41 result.UnpackError = err.Error() 44 42 fillCommandErrors(result, commands, err.Error()) 45 43 46 - return "", nil, false 47 - } 48 - 49 - pending, err := ingest.Ingest( 50 - req.Pack, 51 - service.opts.Algorithm, 52 - ingest.Options{ 53 - FixThin: true, 54 - WriteRev: true, 55 - Base: service.opts.ExistingObjects, 56 - Progress: service.opts.Progress, 57 - }, 58 - ) 59 - if err != nil { 60 - utils.BestEffortFprintf(service.opts.Progress, "unpack failed: %v.\n", err) 61 - 62 - result.UnpackError = err.Error() 63 - fillCommandErrors(result, commands, err.Error()) 64 - 65 - return "", nil, false 66 - } 67 - 68 - if pending.Header().ObjectCount == 0 { 69 - discarded, err := pending.Discard() 70 - if err != nil { 71 - utils.BestEffortFprintf(service.opts.Progress, "unpack failed: %v.\n", err) 72 - 73 - result.UnpackError = err.Error() 74 - fillCommandErrors(result, commands, err.Error()) 75 - 76 - return "", nil, false 77 - } 78 - 79 - result.Ingest = &ingest.Result{ 80 - PackHash: discarded.PackHash, 81 - ObjectCount: discarded.ObjectCount, 82 - } 83 - 84 - utils.BestEffortFprintf( 85 - service.opts.Progress, 86 - "unpacking: done (%d objects, %s).\n", 87 - discarded.ObjectCount, 88 - discarded.PackHash, 89 - ) 90 - 91 - return "", nil, true 44 + return nil, false 92 45 } 93 46 94 47 utils.BestEffortFprintf(service.opts.Progress, "creating quarantine...\r") 95 48 96 - quarantineName, quarantineRoot, err := service.createQuarantineRoot() 97 - if err != nil { 98 - utils.BestEffortFprintf(service.opts.Progress, "unpack failed: %v.\n", err) 99 - 100 - result.UnpackError = err.Error() 101 - fillCommandErrors(result, commands, err.Error()) 102 - 103 - return "", nil, false 104 - } 105 - 106 - quarantinePackRoot, err := service.openQuarantinePackRoot(quarantineRoot) 49 + quarantine, err := service.opts.ObjectIngress.BeginQuarantine(objectstore.QuarantineOptions{}) 107 50 if err != nil { 108 51 utils.BestEffortFprintf(service.opts.Progress, "unpack failed: %v.\n", err) 109 52 110 53 result.UnpackError = err.Error() 111 54 fillCommandErrors(result, commands, err.Error()) 112 55 113 - _ = quarantineRoot.Close() 114 - _ = service.opts.ObjectsRoot.RemoveAll(quarantineName) 115 - 116 - return "", nil, false 56 + return nil, false 117 57 } 118 58 119 59 utils.BestEffortFprintf(service.opts.Progress, "creating quarantine: done.\n") 120 60 utils.BestEffortFprintf(service.opts.Progress, "unpacking...\r") 121 61 122 - ingested, err := pending.Continue(quarantinePackRoot) 123 - 124 - _ = quarantinePackRoot.Close() 125 - 62 + err = quarantine.WritePack(req.Pack, objectstore.PackWriteOptions{ 63 + ThinBase: service.opts.ExistingObjects, 64 + Progress: service.opts.Progress, 65 + RequireTrailingEOF: false, 66 + }) 126 67 if err != nil { 127 68 utils.BestEffortFprintf(service.opts.Progress, "unpack failed: %v.\n", err) 128 69 129 70 result.UnpackError = err.Error() 130 71 fillCommandErrors(result, commands, err.Error()) 131 72 132 - _ = quarantineRoot.Close() 133 - _ = service.opts.ObjectsRoot.RemoveAll(quarantineName) 73 + _ = quarantine.Discard() 134 74 135 - return "", nil, false 75 + return nil, false 136 76 } 137 77 138 - utils.BestEffortFprintf(service.opts.Progress, "unpacking: done (%d objects, %s).\n", ingested.ObjectCount, ingested.PackHash) 139 - 140 - result.Ingest = &ingested 78 + utils.BestEffortFprintf(service.opts.Progress, "unpacking: done.\n") 141 79 142 - return quarantineName, quarantineRoot, true 80 + return quarantine, true 143 81 }
+9 -17
network/receivepack/service/options.go
··· 1 1 package service 2 2 3 3 import ( 4 - "io/fs" 5 - "os" 6 - 7 4 "codeberg.org/lindenii/furgit/common/iowrap" 8 5 commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" 9 6 objectid "codeberg.org/lindenii/furgit/object/id" ··· 11 8 refstore "codeberg.org/lindenii/furgit/ref/store" 12 9 ) 13 10 14 - type PromotedObjectPermissions struct { 15 - DirMode fs.FileMode 16 - FileMode fs.FileMode 17 - } 18 - 19 11 // Options configures one protocol-independent receive-pack service. 20 12 // 21 13 // Service borrows all configured dependencies. 22 14 // 23 15 // Refs and ExistingObjects are required and must be non-nil. 24 - // ObjectsRoot is required if Execute may need to ingest or promote a pack. 25 - // Progress, Hook, and HookIO are optional; when provided they are also 16 + // ObjectIngress is required if Execute may need to ingest or quarantine a 17 + // pack. 18 + // CommitGraph, Progress, Hook, and HookIO are optional; when provided they are also 26 19 // borrowed for the duration of Execute. 27 20 type Options struct { 28 21 Algorithm objectid.Algorithm ··· 31 24 refstore.TransactionalStore 32 25 refstore.BatchStore 33 26 } 34 - ExistingObjects objectstore.Reader 35 - CommitGraph *commitgraphread.Reader 36 - ObjectsRoot *os.Root 37 - Progress iowrap.WriteFlusher 38 - PromotedObjectPermissions *PromotedObjectPermissions 39 - Hook Hook 40 - HookIO HookIO 27 + ExistingObjects objectstore.Reader 28 + ObjectIngress objectstore.Quarantiner 29 + CommitGraph *commitgraphread.Reader 30 + Progress iowrap.WriteFlusher 31 + Hook Hook 32 + HookIO HookIO 41 33 }
-274
network/receivepack/service/quarantine.go
··· 1 - package service 2 - 3 - import ( 4 - "bytes" 5 - "crypto/rand" 6 - "errors" 7 - "fmt" 8 - "io" 9 - "io/fs" 10 - "os" 11 - "path" 12 - "slices" 13 - ) 14 - 15 - // createQuarantineRoot creates one per-push quarantine directory beneath the 16 - // permanent objects root. 17 - // 18 - // It returns both the quarantine directory name relative to ObjectsRoot and an 19 - // opened root for that directory. Callers use the name for later promotion or 20 - // removal relative to ObjectsRoot, and use the opened root for capability-based 21 - // access within the quarantine itself. 22 - func (service *Service) createQuarantineRoot() (string, *os.Root, error) { 23 - name := "tmp_objdir-incoming-" + rand.Text() 24 - 25 - err := service.opts.ObjectsRoot.Mkdir(name, 0o700) 26 - if err != nil { 27 - return "", nil, err 28 - } 29 - 30 - root, err := service.opts.ObjectsRoot.OpenRoot(name) 31 - if err != nil { 32 - _ = service.opts.ObjectsRoot.RemoveAll(name) 33 - 34 - return "", nil, err 35 - } 36 - 37 - return name, root, nil 38 - } 39 - 40 - func (service *Service) openQuarantinePackRoot(quarantineRoot *os.Root) (*os.Root, error) { 41 - err := quarantineRoot.Mkdir("pack", 0o755) 42 - if err != nil && !os.IsExist(err) { 43 - return nil, err 44 - } 45 - 46 - return quarantineRoot.OpenRoot("pack") 47 - } 48 - 49 - func (service *Service) promoteQuarantine(quarantineName string, quarantineRoot *os.Root) error { 50 - if quarantineName == "" || quarantineRoot == nil { 51 - return nil 52 - } 53 - 54 - return service.promoteQuarantineDir(quarantineName, quarantineRoot, ".") 55 - } 56 - 57 - func (service *Service) promoteQuarantineDir(quarantineName string, quarantineRoot *os.Root, rel string) error { 58 - entries, err := fs.ReadDir(quarantineRoot.FS(), rel) 59 - if err != nil && !os.IsNotExist(err) { 60 - return err 61 - } 62 - 63 - slices.SortFunc(entries, func(left, right fs.DirEntry) int { 64 - return packCopyPriority(left.Name()) - packCopyPriority(right.Name()) 65 - }) 66 - 67 - for _, entry := range entries { 68 - childRel := entry.Name() 69 - if rel != "." { 70 - childRel = path.Join(rel, entry.Name()) 71 - } 72 - 73 - if entry.IsDir() { 74 - err = service.opts.ObjectsRoot.Mkdir(childRel, 0o755) 75 - if err != nil && !os.IsExist(err) { 76 - return err 77 - } 78 - 79 - err = service.applyPromotedDirectoryPermissions(childRel) 80 - if err != nil { 81 - return err 82 - } 83 - 84 - err = service.promoteQuarantineDir(quarantineName, quarantineRoot, childRel) 85 - if err != nil { 86 - return err 87 - } 88 - 89 - continue 90 - } 91 - 92 - err = finalizeQuarantineFile( 93 - service.opts.ObjectsRoot, 94 - path.Join(quarantineName, childRel), 95 - childRel, 96 - isLooseObjectShardPath(rel), 97 - service.opts.PromotedObjectPermissions, 98 - ) 99 - if err == nil { 100 - continue 101 - } 102 - 103 - return err 104 - } 105 - 106 - return nil 107 - } 108 - 109 - func packCopyPriority(name string) int { 110 - if !pathHasPackPrefix(name) { 111 - return 0 112 - } 113 - 114 - switch { 115 - case path.Ext(name) == ".keep": 116 - return 1 117 - case path.Ext(name) == ".pack": 118 - return 2 119 - case path.Ext(name) == ".rev": 120 - return 3 121 - case path.Ext(name) == ".idx": 122 - return 4 123 - default: 124 - return 5 125 - } 126 - } 127 - 128 - func pathHasPackPrefix(name string) bool { 129 - return len(name) >= 4 && name[:4] == "pack" 130 - } 131 - 132 - func isLooseObjectShardPath(rel string) bool { 133 - return len(rel) == 2 && isHex(rel[0]) && isHex(rel[1]) 134 - } 135 - 136 - func isHex(ch byte) bool { 137 - return ('0' <= ch && ch <= '9') || ('a' <= ch && ch <= 'f') || ('A' <= ch && ch <= 'F') 138 - } 139 - 140 - func (service *Service) applyPromotedDirectoryPermissions(name string) error { 141 - if service.opts.PromotedObjectPermissions == nil { 142 - return nil 143 - } 144 - 145 - return service.opts.ObjectsRoot.Chmod(name, service.opts.PromotedObjectPermissions.DirMode) 146 - } 147 - 148 - func applyPromotedFilePermissions( 149 - root *os.Root, 150 - name string, 151 - perms *PromotedObjectPermissions, 152 - ) error { 153 - if perms == nil { 154 - return nil 155 - } 156 - 157 - return root.Chmod(name, perms.FileMode) 158 - } 159 - 160 - func finalizeQuarantineFile( 161 - root *os.Root, 162 - src, dst string, 163 - skipCollisionCheck bool, 164 - perms *PromotedObjectPermissions, 165 - ) error { 166 - const maxVanishedRetries = 5 167 - 168 - for retries := 0; ; retries++ { 169 - err := root.Link(src, dst) 170 - switch { 171 - case err == nil: 172 - _ = root.Remove(src) 173 - 174 - return applyPromotedFilePermissions(root, dst, perms) 175 - case !errors.Is(err, fs.ErrExist): 176 - _, statErr := root.Stat(dst) 177 - switch { 178 - case statErr == nil: 179 - err = fs.ErrExist 180 - case errors.Is(statErr, fs.ErrNotExist): 181 - renameErr := root.Rename(src, dst) 182 - if renameErr == nil { 183 - return applyPromotedFilePermissions(root, dst, perms) 184 - } 185 - 186 - err = renameErr 187 - default: 188 - _ = root.Remove(src) 189 - 190 - return statErr 191 - } 192 - } 193 - 194 - if !errors.Is(err, fs.ErrExist) { 195 - _ = root.Remove(src) 196 - 197 - return fmt.Errorf("promote quarantine %q -> %q: %w", src, dst, err) 198 - } 199 - 200 - if skipCollisionCheck { 201 - _ = root.Remove(src) 202 - 203 - return applyPromotedFilePermissions(root, dst, perms) 204 - } 205 - 206 - equal, vanished, cmpErr := compareRootFiles(root, src, dst) 207 - if vanished { 208 - if retries >= maxVanishedRetries { 209 - return fmt.Errorf("promote quarantine %q -> %q: destination repeatedly vanished", src, dst) 210 - } 211 - 212 - continue 213 - } 214 - 215 - if cmpErr != nil { 216 - return cmpErr 217 - } 218 - 219 - if !equal { 220 - return fmt.Errorf("promote quarantine %q -> %q: files differ in contents", src, dst) 221 - } 222 - 223 - _ = root.Remove(src) 224 - 225 - return applyPromotedFilePermissions(root, dst, perms) 226 - } 227 - } 228 - 229 - func compareRootFiles(root *os.Root, left, right string) (equal bool, vanished bool, err error) { 230 - leftFile, err := root.Open(left) 231 - if err != nil { 232 - return false, false, err 233 - } 234 - 235 - defer func() { 236 - _ = leftFile.Close() 237 - }() 238 - 239 - rightFile, err := root.Open(right) 240 - if err != nil { 241 - if errors.Is(err, fs.ErrNotExist) { 242 - return false, true, nil 243 - } 244 - 245 - return false, false, err 246 - } 247 - 248 - defer func() { 249 - _ = rightFile.Close() 250 - }() 251 - 252 - var leftBuf, rightBuf [4096]byte 253 - 254 - for { 255 - leftN, leftErr := leftFile.Read(leftBuf[:]) 256 - rightN, rightErr := rightFile.Read(rightBuf[:]) 257 - 258 - if leftErr != nil && !errors.Is(leftErr, io.EOF) { 259 - return false, false, leftErr 260 - } 261 - 262 - if rightErr != nil && !errors.Is(rightErr, io.EOF) { 263 - return false, false, rightErr 264 - } 265 - 266 - if leftN != rightN || !bytes.Equal(leftBuf[:leftN], rightBuf[:rightN]) { 267 - return false, false, nil 268 - } 269 - 270 - if leftErr != nil || rightErr != nil { 271 - return true, false, nil 272 - } 273 - } 274 - }
-184
network/receivepack/service/quarantine_test.go
··· 1 - package service //nolint:testpackage 2 - 3 - // because we need access to quarantine internals 4 - 5 - import ( 6 - "os" 7 - "path" 8 - "testing" 9 - 10 - objectid "codeberg.org/lindenii/furgit/object/id" 11 - "codeberg.org/lindenii/furgit/object/store/memory" 12 - ) 13 - 14 - type quarantineFixture struct { 15 - svc *Service 16 - objectsRoot *os.Root 17 - quarantineName string 18 - quarantineRoot *os.Root 19 - } 20 - 21 - func newQuarantineFixture(tb testing.TB, opts Options) *quarantineFixture { 22 - tb.Helper() 23 - 24 - objectsRoot, err := os.OpenRoot(tb.TempDir()) 25 - if err != nil { 26 - tb.Fatalf("os.OpenRoot: %v", err) 27 - } 28 - 29 - tb.Cleanup(func() { 30 - _ = objectsRoot.Close() 31 - }) 32 - 33 - opts.Algorithm = objectid.AlgorithmSHA1 34 - opts.ExistingObjects = memory.New(objectid.AlgorithmSHA1) 35 - opts.ObjectsRoot = objectsRoot 36 - 37 - svc := New(opts) 38 - 39 - quarantineName, quarantineRoot, err := svc.createQuarantineRoot() 40 - if err != nil { 41 - tb.Fatalf("createQuarantineRoot: %v", err) 42 - } 43 - 44 - tb.Cleanup(func() { 45 - _ = quarantineRoot.Close() 46 - _ = objectsRoot.RemoveAll(quarantineName) 47 - }) 48 - 49 - return &quarantineFixture{ 50 - svc: svc, 51 - objectsRoot: objectsRoot, 52 - quarantineName: quarantineName, 53 - quarantineRoot: quarantineRoot, 54 - } 55 - } 56 - 57 - func writeMatchingPromotedFile( 58 - tb testing.TB, 59 - quarantineRoot, objectsRoot *os.Root, 60 - dir, name, payload string, 61 - ) { 62 - tb.Helper() 63 - 64 - err := quarantineRoot.Mkdir(dir, 0o755) 65 - if err != nil { 66 - tb.Fatalf("Mkdir(%s): %v", dir, err) 67 - } 68 - 69 - err = objectsRoot.Mkdir(dir, 0o755) 70 - if err != nil { 71 - tb.Fatalf("Mkdir(dst %s): %v", dir, err) 72 - } 73 - 74 - rel := path.Join(dir, name) 75 - 76 - err = quarantineRoot.WriteFile(rel, []byte(payload), 0o644) 77 - if err != nil { 78 - tb.Fatalf("WriteFile(quarantine %s): %v", rel, err) 79 - } 80 - 81 - err = objectsRoot.WriteFile(rel, []byte(payload), 0o644) 82 - if err != nil { 83 - tb.Fatalf("WriteFile(permanent %s): %v", rel, err) 84 - } 85 - } 86 - 87 - func TestPromoteQuarantineAppliesConfiguredPermissions(t *testing.T) { 88 - t.Parallel() 89 - 90 - fx := newQuarantineFixture(t, Options{ 91 - PromotedObjectPermissions: &PromotedObjectPermissions{ 92 - DirMode: 0o751, 93 - FileMode: 0o640, 94 - }, 95 - }) 96 - 97 - err := fx.quarantineRoot.Mkdir("ab", 0o700) 98 - if err != nil { 99 - t.Fatalf("Mkdir(ab): %v", err) 100 - } 101 - 102 - err = fx.quarantineRoot.WriteFile(path.Join("ab", "cdef"), []byte("payload"), 0o600) 103 - if err != nil { 104 - t.Fatalf("WriteFile(quarantine loose): %v", err) 105 - } 106 - 107 - err = fx.svc.promoteQuarantine(fx.quarantineName, fx.quarantineRoot) 108 - if err != nil { 109 - t.Fatalf("promoteQuarantine: %v", err) 110 - } 111 - 112 - dirInfo, err := fx.objectsRoot.Stat("ab") 113 - if err != nil { 114 - t.Fatalf("Stat(ab): %v", err) 115 - } 116 - 117 - if got := dirInfo.Mode().Perm(); got != 0o751 { 118 - t.Fatalf("dir mode = %o, want 751", got) 119 - } 120 - 121 - fileInfo, err := fx.objectsRoot.Stat(path.Join("ab", "cdef")) 122 - if err != nil { 123 - t.Fatalf("Stat(ab/cdef): %v", err) 124 - } 125 - 126 - if got := fileInfo.Mode().Perm(); got != 0o640 { 127 - t.Fatalf("file mode = %o, want 640", got) 128 - } 129 - } 130 - 131 - func TestPromoteQuarantineTreatsExistingLooseObjectAsSuccess(t *testing.T) { 132 - t.Parallel() 133 - 134 - fx := newQuarantineFixture(t, Options{}) 135 - writeMatchingPromotedFile(t, fx.quarantineRoot, fx.objectsRoot, "ab", "cdef", "same object bytes") 136 - 137 - err := fx.svc.promoteQuarantine(fx.quarantineName, fx.quarantineRoot) 138 - if err != nil { 139 - t.Fatalf("promoteQuarantine: %v", err) 140 - } 141 - } 142 - 143 - func TestPromoteQuarantineRejectsDifferentExistingPackFile(t *testing.T) { 144 - t.Parallel() 145 - 146 - fx := newQuarantineFixture(t, Options{}) 147 - 148 - err := fx.quarantineRoot.Mkdir("pack", 0o755) 149 - if err != nil { 150 - t.Fatalf("Mkdir(pack): %v", err) 151 - } 152 - 153 - err = fx.objectsRoot.Mkdir("pack", 0o755) 154 - if err != nil { 155 - t.Fatalf("Mkdir(dst pack): %v", err) 156 - } 157 - 158 - err = fx.quarantineRoot.WriteFile(path.Join("pack", "pack-a.pack"), []byte("new bytes"), 0o644) 159 - if err != nil { 160 - t.Fatalf("WriteFile(quarantine pack): %v", err) 161 - } 162 - 163 - err = fx.objectsRoot.WriteFile(path.Join("pack", "pack-a.pack"), []byte("old bytes"), 0o644) 164 - if err != nil { 165 - t.Fatalf("WriteFile(permanent pack): %v", err) 166 - } 167 - 168 - err = fx.svc.promoteQuarantine(fx.quarantineName, fx.quarantineRoot) 169 - if err == nil { 170 - t.Fatal("promoteQuarantine unexpectedly succeeded") 171 - } 172 - } 173 - 174 - func TestPromoteQuarantineAcceptsMatchingExistingPackFile(t *testing.T) { 175 - t.Parallel() 176 - 177 - fx := newQuarantineFixture(t, Options{}) 178 - writeMatchingPromotedFile(t, fx.quarantineRoot, fx.objectsRoot, "pack", "pack-a.pack", "identical pack bytes") 179 - 180 - err := fx.svc.promoteQuarantine(fx.quarantineName, fx.quarantineRoot) 181 - if err != nil { 182 - t.Fatalf("promoteQuarantine: %v", err) 183 - } 184 - }
-5
network/receivepack/service/result.go
··· 1 1 package service 2 2 3 - import ( 4 - "codeberg.org/lindenii/furgit/format/packfile/ingest" 5 - ) 6 - 7 3 // Result is one receive-pack execution result. 8 4 type Result struct { 9 5 UnpackError string 10 6 Commands []CommandResult 11 - Ingest *ingest.Result 12 7 Planned []PlannedUpdate 13 8 Applied bool 14 9 }
+1 -81
network/receivepack/service/run_hook.go
··· 2 2 3 3 import ( 4 4 "context" 5 - "os" 6 5 7 6 "codeberg.org/lindenii/furgit/internal/utils" 8 7 objectstore "codeberg.org/lindenii/furgit/object/store" 9 - "codeberg.org/lindenii/furgit/object/store/loose" 10 - objectmix "codeberg.org/lindenii/furgit/object/store/mix" 11 - "codeberg.org/lindenii/furgit/object/store/packed" 12 8 ) 13 9 14 10 func (service *Service) runHook( 15 11 ctx context.Context, 16 12 req *Request, 17 13 commands []Command, 18 - quarantineName string, 14 + quarantinedObjects objectstore.Reader, 19 15 ) ( 20 16 allowedCommands []Command, 21 17 allowedIndices []int, ··· 36 32 } 37 33 38 34 utils.BestEffortFprintf(service.opts.Progress, "running hooks...\r") 39 - 40 - quarantinedObjects := service.opts.ExistingObjects 41 - 42 - var ( 43 - quarantineObjectsStore objectstore.Reader 44 - quarantineLooseStore *loose.Store 45 - quarantinePackedStore *packed.Store 46 - quarantineLooseRoot *os.Root 47 - quarantinePackRoot *os.Root 48 - err error 49 - ) 50 - 51 - //nolint:nestif 52 - if quarantineName != "" { 53 - quarantineLooseRoot, err = service.opts.ObjectsRoot.OpenRoot(quarantineName) 54 - if err != nil { 55 - utils.BestEffortFprintf(service.opts.Progress, "running hooks: failed: %v.\n", err) 56 - 57 - return nil, nil, nil, false, err.Error() 58 - } 59 - 60 - quarantineLooseStore, err = loose.New(quarantineLooseRoot, service.opts.Algorithm) 61 - if err != nil { 62 - _ = quarantineLooseRoot.Close() 63 - 64 - utils.BestEffortFprintf(service.opts.Progress, "running hooks: failed: %v.\n", err) 65 - 66 - return nil, nil, nil, false, err.Error() 67 - } 68 - 69 - quarantinedObjects = quarantineLooseStore 70 - 71 - quarantinePackRoot, err = quarantineLooseRoot.OpenRoot("pack") 72 - if err == nil { 73 - var packedErr error 74 - 75 - quarantinePackedStore, packedErr = packed.New(quarantinePackRoot, service.opts.Algorithm, packed.Options{}) 76 - if packedErr != nil { 77 - _ = quarantineLooseStore.Close() 78 - _ = quarantinePackRoot.Close() 79 - _ = quarantineLooseRoot.Close() 80 - 81 - utils.BestEffortFprintf(service.opts.Progress, "running hooks: failed: %v.\n", packedErr) 82 - 83 - return nil, nil, nil, false, packedErr.Error() 84 - } 85 - 86 - quarantineObjectsStore = objectmix.New(quarantineLooseStore, quarantinePackedStore) 87 - quarantinedObjects = quarantineObjectsStore 88 - } else if !os.IsNotExist(err) { 89 - _ = quarantineLooseStore.Close() 90 - _ = quarantineLooseRoot.Close() 91 - 92 - utils.BestEffortFprintf(service.opts.Progress, "running hooks: failed: %v.\n", err) 93 - 94 - return nil, nil, nil, false, err.Error() 95 - } 96 - 97 - defer func() { 98 - if quarantinePackedStore != nil { 99 - _ = quarantinePackedStore.Close() 100 - } 101 - 102 - if quarantineLooseStore != nil { 103 - _ = quarantineLooseStore.Close() 104 - } 105 - 106 - if quarantinePackRoot != nil { 107 - _ = quarantinePackRoot.Close() 108 - } 109 - 110 - if quarantineLooseRoot != nil { 111 - _ = quarantineLooseRoot.Close() 112 - } 113 - }() 114 - } 115 35 116 36 decisions, err := service.opts.Hook(ctx, HookRequest{ 117 37 Refs: service.opts.Refs,
+54 -22
network/receivepack/service/service_test.go
··· 2 2 3 3 import ( 4 4 "context" 5 - "io/fs" 6 5 "os" 7 6 "strings" 8 7 "testing" ··· 10 9 "codeberg.org/lindenii/furgit/internal/testgit" 11 10 "codeberg.org/lindenii/furgit/network/receivepack/service" 12 11 objectid "codeberg.org/lindenii/furgit/object/id" 12 + objectstore "codeberg.org/lindenii/furgit/object/store" 13 + objectdual "codeberg.org/lindenii/furgit/object/store/dual" 14 + objectloose "codeberg.org/lindenii/furgit/object/store/loose" 13 15 "codeberg.org/lindenii/furgit/object/store/memory" 16 + objectpacked "codeberg.org/lindenii/furgit/object/store/packed" 14 17 ) 15 18 16 - func TestExecutePackExpectedWithoutObjectsRoot(t *testing.T) { 19 + func TestExecutePackExpectedWithoutObjectIngress(t *testing.T) { 17 20 t.Parallel() 18 21 19 22 //nolint:thelper ··· 39 42 t.Fatalf("Execute: %v", err) 40 43 } 41 44 42 - if result.UnpackError != "objects root not configured" { 45 + if result.UnpackError != "object ingress not configured" { 43 46 t.Fatalf("unexpected unpack error %q", result.UnpackError) 44 47 } 45 48 }) 46 49 } 47 50 48 - func TestExecuteRemovesDerivedQuarantineAfterIngestFailure(t *testing.T) { 51 + func TestExecuteDiscardedQuarantineAfterIngestFailure(t *testing.T) { 49 52 t.Parallel() 50 53 51 54 //nolint:thelper ··· 53 56 t.Parallel() 54 57 55 58 store := memory.New(algo) 56 - objectsDir := t.TempDir() 57 - 58 - objectsRoot, err := os.OpenRoot(objectsDir) 59 - if err != nil { 60 - t.Fatalf("os.OpenRoot: %v", err) 61 - } 62 - 63 - t.Cleanup(func() { 64 - _ = objectsRoot.Close() 65 - }) 59 + objectIngress := newDualIngress(t, algo) 66 60 67 61 svc := service.New(service.Options{ 68 62 Algorithm: algo, 69 63 ExistingObjects: store, 70 - ObjectsRoot: objectsRoot, 64 + ObjectIngress: objectIngress, 71 65 }) 72 66 73 67 result, err := svc.Execute(context.Background(), &service.Request{ ··· 86 80 if result.UnpackError == "" { 87 81 t.Fatal("Execute returned empty unpack error for invalid pack") 88 82 } 83 + }) 84 + } 89 85 90 - entries, err := fs.ReadDir(objectsRoot.FS(), ".") 91 - if err != nil { 92 - t.Fatalf("fs.ReadDir: %v", err) 93 - } 86 + func newDualIngress(tb testing.TB, algo objectid.Algorithm) objectstore.Quarantiner { 87 + tb.Helper() 94 88 95 - if len(entries) != 0 { 96 - t.Fatalf("objects root still has entries after failed ingest: %d", len(entries)) 97 - } 89 + objectsRoot, err := os.OpenRoot(tb.TempDir()) 90 + if err != nil { 91 + tb.Fatalf("os.OpenRoot: %v", err) 92 + } 93 + 94 + tb.Cleanup(func() { 95 + _ = objectsRoot.Close() 98 96 }) 97 + 98 + err = objectsRoot.Mkdir("pack", 0o755) 99 + if err != nil { 100 + tb.Fatalf("Mkdir(pack): %v", err) 101 + } 102 + 103 + packRoot, err := objectsRoot.OpenRoot("pack") 104 + if err != nil { 105 + tb.Fatalf("OpenRoot(pack): %v", err) 106 + } 107 + 108 + tb.Cleanup(func() { 109 + _ = packRoot.Close() 110 + }) 111 + 112 + looseStore, err := objectloose.New(objectsRoot, algo) 113 + if err != nil { 114 + tb.Fatalf("loose.New: %v", err) 115 + } 116 + 117 + tb.Cleanup(func() { 118 + _ = looseStore.Close() 119 + }) 120 + 121 + packedStore, err := objectpacked.New(packRoot, algo, objectpacked.Options{WriteRev: true}) 122 + if err != nil { 123 + tb.Fatalf("packed.New: %v", err) 124 + } 125 + 126 + tb.Cleanup(func() { 127 + _ = packedStore.Close() 128 + }) 129 + 130 + return objectdual.New(looseStore, packedStore) 99 131 }