Fast implementation of Git in pure Go codeberg.org/lindenii/furgit
git go
6
fork

Configure Feed

Select the types of activity you want to include in your feed.

object/store/dual: Add a basic dual composr

Runxi Yu c3fe7af6 d298ab6c

+620
+8
object/store/dual/doc.go
··· 1 + // Package dual provides one logical object store backed by separate object-wise 2 + // and pack-wise stores. 3 + // 4 + // Dual composes a store that handles individual object writes with a store that 5 + // handles pack-wise writes, while exposing one mixed reader over both. 6 + // Coordinated quarantine operations span both stores, but quarantine promotion 7 + // is non-atomic. 8 + package dual
+35
object/store/dual/dual.go
··· 1 + package dual 2 + 3 + import objectstore "codeberg.org/lindenii/furgit/object/store" 4 + 5 + type objectSide interface { 6 + objectstore.Reader 7 + objectstore.ObjectWriter 8 + objectstore.ObjectQuarantiner 9 + } 10 + 11 + type packSide interface { 12 + objectstore.Reader 13 + objectstore.PackWriter 14 + objectstore.PackQuarantiner 15 + } 16 + 17 + // Dual composes one object-wise store and one pack-wise store into one logical 18 + // object store. 19 + // 20 + // Reads are served from the combined object reader of both stores. Individual 21 + // object writes are routed to the object-wise store, and pack writes are routed 22 + // to the pack-wise store. Coordinated quarantines go across both stores. 23 + type Dual struct { 24 + object objectSide 25 + pack packSide 26 + reader objectstore.Reader 27 + } 28 + 29 + var ( 30 + _ objectstore.Reader = (*Dual)(nil) 31 + _ objectstore.ObjectWriter = (*Dual)(nil) 32 + _ objectstore.PackWriter = (*Dual)(nil) 33 + _ objectstore.ObjectQuarantiner = (*Dual)(nil) 34 + _ objectstore.PackQuarantiner = (*Dual)(nil) 35 + )
+263
object/store/dual/dual_test.go
··· 1 + package dual_test 2 + 3 + import ( 4 + "bytes" 5 + "os" 6 + "path/filepath" 7 + "strings" 8 + "testing" 9 + 10 + "codeberg.org/lindenii/furgit/internal/testgit" 11 + objectid "codeberg.org/lindenii/furgit/object/id" 12 + objectstore "codeberg.org/lindenii/furgit/object/store" 13 + "codeberg.org/lindenii/furgit/object/store/dual" 14 + "codeberg.org/lindenii/furgit/object/store/loose" 15 + "codeberg.org/lindenii/furgit/object/store/packed" 16 + objecttype "codeberg.org/lindenii/furgit/object/type" 17 + ) 18 + 19 + func fixturePath(t *testing.T, algo objectid.Algorithm, name string) string { 20 + t.Helper() 21 + 22 + return filepath.Join("..", "packed", "internal", "ingest", "testdata", "fixtures", algo.String(), name) 23 + } 24 + 25 + func fixtureBytes(t *testing.T, algo objectid.Algorithm, name string) []byte { 26 + t.Helper() 27 + 28 + path := fixturePath(t, algo, name) 29 + dir := filepath.Dir(path) 30 + base := filepath.Base(path) 31 + 32 + root, err := os.OpenRoot(dir) 33 + if err != nil { 34 + t.Fatalf("open fixture root %q: %v", dir, err) 35 + } 36 + 37 + defer func() { 38 + err := root.Close() 39 + if err != nil { 40 + t.Fatalf("close fixture root %q: %v", dir, err) 41 + } 42 + }() 43 + 44 + data, err := root.ReadFile(base) 45 + if err != nil { 46 + t.Fatalf("read fixture %q: %v", base, err) 47 + } 48 + 49 + return data 50 + } 51 + 52 + func fixtureMetadata(t *testing.T, algo objectid.Algorithm) map[string]string { 53 + t.Helper() 54 + 55 + data := fixtureBytes(t, algo, "METADATA.txt") 56 + out := make(map[string]string) 57 + 58 + for line := range strings.SplitSeq(strings.TrimSpace(string(data)), "\n") { 59 + line = strings.TrimSpace(line) 60 + if line == "" { 61 + continue 62 + } 63 + 64 + key, value, ok := strings.Cut(line, "=") 65 + if !ok { 66 + t.Fatalf("invalid fixture metadata line %q", line) 67 + } 68 + 69 + out[strings.TrimSpace(key)] = strings.TrimSpace(value) 70 + } 71 + 72 + return out 73 + } 74 + 75 + func fixtureOID(t *testing.T, algo objectid.Algorithm, key string) objectid.ObjectID { 76 + t.Helper() 77 + 78 + meta := fixtureMetadata(t, algo) 79 + hex, ok := meta[key] 80 + if !ok { 81 + t.Fatalf("missing fixture metadata key %q", key) 82 + } 83 + 84 + id, err := objectid.ParseHex(algo, hex) 85 + if err != nil { 86 + t.Fatalf("parse fixture metadata oid %q: %v", hex, err) 87 + } 88 + 89 + return id 90 + } 91 + 92 + func newDualStore(t *testing.T, repo *testgit.TestRepo, algo objectid.Algorithm) *dual.Dual { 93 + t.Helper() 94 + 95 + objectsRoot := repo.OpenObjectsRoot(t) 96 + looseStore, err := loose.New(objectsRoot, algo) 97 + if err != nil { 98 + t.Fatalf("loose.New: %v", err) 99 + } 100 + 101 + packRoot := repo.OpenPackRoot(t) 102 + packedStore, err := packed.New(packRoot, algo, packed.Options{WriteRev: true}) 103 + if err != nil { 104 + t.Fatalf("packed.New: %v", err) 105 + } 106 + 107 + return dual.New(looseStore, packedStore) 108 + } 109 + 110 + func TestDualReadsWritesAndQuarantine(t *testing.T) { 111 + t.Parallel() 112 + 113 + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { 114 + head := fixtureOID(t, algo, "head") 115 + packBytes := fixtureBytes(t, algo, "nonthin.pack") 116 + 117 + repo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) 118 + store := newDualStore(t, repo, algo) 119 + 120 + quarantiner, ok := any(store).(objectstore.PackQuarantiner) 121 + if !ok { 122 + t.Fatal("dual does not implement PackQuarantiner") 123 + } 124 + 125 + quarantine, err := quarantiner.BeginPackQuarantine(objectstore.PackQuarantineOptions{}) 126 + if err != nil { 127 + t.Fatalf("BeginPackQuarantine: %v", err) 128 + } 129 + 130 + err = quarantine.WritePack(bytes.NewReader(packBytes), objectstore.PackWriteOptions{RequireTrailingEOF: true}) 131 + if err != nil { 132 + t.Fatalf("quarantine.WritePack: %v", err) 133 + } 134 + 135 + objectQ, ok := any(quarantine).(objectstore.ObjectQuarantine) 136 + if !ok { 137 + t.Fatal("pack quarantine does not also implement ObjectQuarantine") 138 + } 139 + 140 + looseContent := []byte("dual quarantine loose object\n") 141 + looseID, err := objectQ.WriteBytesContent(objecttype.TypeBlob, looseContent) 142 + if err != nil { 143 + t.Fatalf("quarantine.WriteBytesContent: %v", err) 144 + } 145 + 146 + ty, _, err := quarantine.ReadHeader(head) 147 + if err != nil { 148 + t.Fatalf("quarantine.ReadHeader(pack): %v", err) 149 + } 150 + 151 + if ty != objecttype.TypeCommit { 152 + t.Fatalf("quarantine.ReadHeader(pack) type = %v, want commit", ty) 153 + } 154 + 155 + ty, got, err := quarantine.ReadBytesContent(looseID) 156 + if err != nil { 157 + t.Fatalf("quarantine.ReadBytesContent(loose): %v", err) 158 + } 159 + 160 + if ty != objecttype.TypeBlob { 161 + t.Fatalf("quarantine.ReadBytesContent(loose) type = %v, want blob", ty) 162 + } 163 + 164 + if !bytes.Equal(got, looseContent) { 165 + t.Fatal("quarantine.ReadBytesContent(loose) mismatch") 166 + } 167 + 168 + _, _, err = store.ReadHeader(head) 169 + if err == nil { 170 + t.Fatal("store.ReadHeader unexpectedly saw quarantined pack object before promote") 171 + } 172 + 173 + _, _, err = store.ReadBytesContent(looseID) 174 + if err == nil { 175 + t.Fatal("store.ReadBytesContent unexpectedly saw quarantined loose object before promote") 176 + } 177 + 178 + err = quarantine.Promote() 179 + if err != nil { 180 + t.Fatalf("quarantine.Promote: %v", err) 181 + } 182 + 183 + err = store.Refresh() 184 + if err != nil { 185 + t.Fatalf("store.Refresh: %v", err) 186 + } 187 + 188 + ty, _, err = store.ReadHeader(head) 189 + if err != nil { 190 + t.Fatalf("store.ReadHeader(pack): %v", err) 191 + } 192 + 193 + if ty != objecttype.TypeCommit { 194 + t.Fatalf("store.ReadHeader(pack) type = %v, want commit", ty) 195 + } 196 + 197 + ty, got, err = store.ReadBytesContent(looseID) 198 + if err != nil { 199 + t.Fatalf("store.ReadBytesContent(loose): %v", err) 200 + } 201 + 202 + if ty != objecttype.TypeBlob { 203 + t.Fatalf("store.ReadBytesContent(loose) type = %v, want blob", ty) 204 + } 205 + 206 + if !bytes.Equal(got, looseContent) { 207 + t.Fatal("store.ReadBytesContent(loose) mismatch") 208 + } 209 + }) 210 + } 211 + 212 + func TestDualQuarantineDiscardDropsBothHalves(t *testing.T) { 213 + t.Parallel() 214 + 215 + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { 216 + head := fixtureOID(t, algo, "head") 217 + packBytes := fixtureBytes(t, algo, "nonthin.pack") 218 + 219 + repo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) 220 + store := newDualStore(t, repo, algo) 221 + 222 + quarantiner := any(store).(objectstore.ObjectQuarantiner) 223 + quarantine, err := quarantiner.BeginObjectQuarantine(objectstore.ObjectQuarantineOptions{}) 224 + if err != nil { 225 + t.Fatalf("BeginObjectQuarantine: %v", err) 226 + } 227 + 228 + packQ, ok := any(quarantine).(objectstore.PackQuarantine) 229 + if !ok { 230 + t.Fatal("object quarantine does not also implement PackQuarantine") 231 + } 232 + 233 + err = packQ.WritePack(bytes.NewReader(packBytes), objectstore.PackWriteOptions{RequireTrailingEOF: true}) 234 + if err != nil { 235 + t.Fatalf("quarantine.WritePack: %v", err) 236 + } 237 + 238 + looseID, err := quarantine.WriteBytesContent(objecttype.TypeBlob, []byte("discarded dual object\n")) 239 + if err != nil { 240 + t.Fatalf("quarantine.WriteBytesContent: %v", err) 241 + } 242 + 243 + err = quarantine.Discard() 244 + if err != nil { 245 + t.Fatalf("quarantine.Discard: %v", err) 246 + } 247 + 248 + err = store.Refresh() 249 + if err != nil { 250 + t.Fatalf("store.Refresh: %v", err) 251 + } 252 + 253 + _, _, err = store.ReadHeader(head) 254 + if err == nil { 255 + t.Fatal("store.ReadHeader unexpectedly saw discarded pack object") 256 + } 257 + 258 + _, _, err = store.ReadBytesContent(looseID) 259 + if err == nil { 260 + t.Fatal("store.ReadBytesContent unexpectedly saw discarded loose object") 261 + } 262 + }) 263 + }
+29
object/store/dual/new.go
··· 1 + package dual 2 + 3 + import ( 4 + objectstore "codeberg.org/lindenii/furgit/object/store" 5 + objectmix "codeberg.org/lindenii/furgit/object/store/mix" 6 + ) 7 + 8 + // New creates one dual object store from borrowed object-wise and pack-wise 9 + // stores. 10 + // 11 + // Labels: Deps-Borrowed, Life-Parent. 12 + func New( 13 + object interface { 14 + objectstore.Reader 15 + objectstore.ObjectWriter 16 + objectstore.ObjectQuarantiner 17 + }, 18 + pack interface { 19 + objectstore.Reader 20 + objectstore.PackWriter 21 + objectstore.PackQuarantiner 22 + }, 23 + ) *Dual { 24 + return &Dual{ 25 + object: object, 26 + pack: pack, 27 + reader: objectmix.New(object, pack), 28 + } 29 + }
+113
object/store/dual/quarantine.go
··· 1 + package dual 2 + 3 + import ( 4 + "io" 5 + 6 + objectid "codeberg.org/lindenii/furgit/object/id" 7 + objectstore "codeberg.org/lindenii/furgit/object/store" 8 + objectmix "codeberg.org/lindenii/furgit/object/store/mix" 9 + objecttype "codeberg.org/lindenii/furgit/object/type" 10 + ) 11 + 12 + // quarantine is one coordinated dual quarantine over both stores. 13 + type quarantine struct { 14 + objectQ objectstore.ObjectQuarantine 15 + packQ objectstore.PackQuarantine 16 + reader objectstore.Reader 17 + } 18 + 19 + var ( 20 + _ objectstore.ObjectQuarantine = (*quarantine)(nil) 21 + _ objectstore.PackQuarantine = (*quarantine)(nil) 22 + ) 23 + 24 + func newQuarantine( 25 + objectQ objectstore.ObjectQuarantine, 26 + packQ objectstore.PackQuarantine, 27 + ) *quarantine { 28 + return &quarantine{ 29 + objectQ: objectQ, 30 + packQ: packQ, 31 + reader: objectmix.New(objectQ, packQ), 32 + } 33 + } 34 + 35 + // ReadBytesFull reads a full serialized object as "type size\0content" from 36 + // either quarantined store. 37 + func (quarantine *quarantine) ReadBytesFull(id objectid.ObjectID) ([]byte, error) { 38 + return quarantine.reader.ReadBytesFull(id) 39 + } 40 + 41 + // ReadBytesContent reads an object's type and content bytes from either 42 + // quarantined store. 43 + func (quarantine *quarantine) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []byte, error) { 44 + return quarantine.reader.ReadBytesContent(id) 45 + } 46 + 47 + // ReadReaderFull reads a full serialized object stream as 48 + // "type size\0content" from either quarantined store. 49 + func (quarantine *quarantine) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error) { 50 + return quarantine.reader.ReadReaderFull(id) 51 + } 52 + 53 + // ReadReaderContent reads an object's type, declared content length, and 54 + // content stream from either quarantined store. 55 + func (quarantine *quarantine) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, int64, io.ReadCloser, error) { 56 + return quarantine.reader.ReadReaderContent(id) 57 + } 58 + 59 + // ReadSize reads an object's declared content length from either quarantined 60 + // store. 61 + func (quarantine *quarantine) ReadSize(id objectid.ObjectID) (int64, error) { 62 + return quarantine.reader.ReadSize(id) 63 + } 64 + 65 + // ReadHeader reads an object's type and declared content length from either 66 + // quarantined store. 67 + func (quarantine *quarantine) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, error) { 68 + return quarantine.reader.ReadHeader(id) 69 + } 70 + 71 + // Refresh refreshes both quarantined stores and the combined quarantined reader. 72 + func (quarantine *quarantine) Refresh() error { 73 + err := quarantine.objectQ.Refresh() 74 + if err != nil { 75 + return err 76 + } 77 + 78 + err = quarantine.packQ.Refresh() 79 + if err != nil { 80 + return err 81 + } 82 + 83 + return quarantine.reader.Refresh() 84 + } 85 + 86 + // WriteReaderContent writes one typed object content stream to the quarantined 87 + // object-wise store. 88 + func (quarantine *quarantine) WriteReaderContent(ty objecttype.Type, size int64, src io.Reader) (objectid.ObjectID, error) { 89 + return quarantine.objectQ.WriteReaderContent(ty, size, src) 90 + } 91 + 92 + // WriteReaderFull writes one full serialized object stream as 93 + // "type size\0content" to the quarantined object-wise store. 94 + func (quarantine *quarantine) WriteReaderFull(src io.Reader) (objectid.ObjectID, error) { 95 + return quarantine.objectQ.WriteReaderFull(src) 96 + } 97 + 98 + // WriteBytesContent writes one typed object content byte slice to the 99 + // quarantined object-wise store. 100 + func (quarantine *quarantine) WriteBytesContent(ty objecttype.Type, content []byte) (objectid.ObjectID, error) { 101 + return quarantine.objectQ.WriteBytesContent(ty, content) 102 + } 103 + 104 + // WriteBytesFull writes one full serialized object byte slice as 105 + // "type size\0content" to the quarantined object-wise store. 106 + func (quarantine *quarantine) WriteBytesFull(raw []byte) (objectid.ObjectID, error) { 107 + return quarantine.objectQ.WriteBytesFull(raw) 108 + } 109 + 110 + // WritePack ingests one pack stream into the quarantined pack-wise store. 111 + func (quarantine *quarantine) WritePack(src io.Reader, opts objectstore.PackWriteOptions) error { 112 + return quarantine.packQ.WritePack(src, opts) 113 + }
+47
object/store/dual/quarantine_begin.go
··· 1 + package dual 2 + 3 + import objectstore "codeberg.org/lindenii/furgit/object/store" 4 + 5 + // TODO: This doesn't actually make sense. We need a combined quarantine. 6 + 7 + // BeginObjectQuarantine creates one coordinated dual quarantine spanning both 8 + // stores and returns it as an object-wise quarantine. 9 + // 10 + // Labels: Deps-Borrowed, Life-Parent, Close-No. 11 + func (dual *Dual) BeginObjectQuarantine(_ objectstore.ObjectQuarantineOptions) (objectstore.ObjectQuarantine, error) { 12 + quarantine, err := dual.beginQuarantine() 13 + if err != nil { 14 + return nil, err 15 + } 16 + 17 + return quarantine, nil 18 + } 19 + 20 + // BeginPackQuarantine creates one coordinated dual quarantine spanning both 21 + // stores and returns it as a pack-wise quarantine. 22 + // 23 + // Labels: Deps-Borrowed, Life-Parent, Close-No. 24 + func (dual *Dual) BeginPackQuarantine(_ objectstore.PackQuarantineOptions) (objectstore.PackQuarantine, error) { 25 + quarantine, err := dual.beginQuarantine() 26 + if err != nil { 27 + return nil, err 28 + } 29 + 30 + return quarantine, nil 31 + } 32 + 33 + func (dual *Dual) beginQuarantine() (*quarantine, error) { 34 + objectQ, err := dual.object.BeginObjectQuarantine(objectstore.ObjectQuarantineOptions{}) 35 + if err != nil { 36 + return nil, err 37 + } 38 + 39 + packQ, err := dual.pack.BeginPackQuarantine(objectstore.PackQuarantineOptions{}) 40 + if err != nil { 41 + _ = objectQ.Discard() 42 + 43 + return nil, err 44 + } 45 + 46 + return newQuarantine(objectQ, packQ), nil 47 + }
+11
object/store/dual/quarantine_discard.go
··· 1 + package dual 2 + 3 + // Discard abandons both quarantine halves and invalidates the receiver. 4 + func (quarantine *quarantine) Discard() error { 5 + err := quarantine.packQ.Discard() 6 + if err != nil { 7 + return err 8 + } 9 + 10 + return quarantine.objectQ.Discard() 11 + }
+13
object/store/dual/quarantine_promote.go
··· 1 + package dual 2 + 3 + // Promote publishes both quarantine halves and invalidates the receiver. 4 + // 5 + // Promotion is coordinated and ordered, but not atomic. 6 + func (quarantine *quarantine) Promote() error { 7 + err := quarantine.packQ.Promote() 8 + if err != nil { 9 + return err 10 + } 11 + 12 + return quarantine.objectQ.Promote() 13 + }
+57
object/store/dual/reader.go
··· 1 + package dual 2 + 3 + import ( 4 + "io" 5 + 6 + objectid "codeberg.org/lindenii/furgit/object/id" 7 + objecttype "codeberg.org/lindenii/furgit/object/type" 8 + ) 9 + 10 + // ReadBytesFull reads a full serialized object as "type size\0content" from 11 + // either store. 12 + func (dual *Dual) ReadBytesFull(id objectid.ObjectID) ([]byte, error) { 13 + return dual.reader.ReadBytesFull(id) 14 + } 15 + 16 + // ReadBytesContent reads an object's type and content bytes from either store. 17 + func (dual *Dual) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []byte, error) { 18 + return dual.reader.ReadBytesContent(id) 19 + } 20 + 21 + // ReadReaderFull reads a full serialized object stream as "type size\0content" 22 + // from either store. 23 + func (dual *Dual) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error) { 24 + return dual.reader.ReadReaderFull(id) 25 + } 26 + 27 + // ReadReaderContent reads an object's type, declared content length, and 28 + // content stream from either store. 29 + func (dual *Dual) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, int64, io.ReadCloser, error) { 30 + return dual.reader.ReadReaderContent(id) 31 + } 32 + 33 + // ReadSize reads an object's declared content length from either store. 34 + func (dual *Dual) ReadSize(id objectid.ObjectID) (int64, error) { 35 + return dual.reader.ReadSize(id) 36 + } 37 + 38 + // ReadHeader reads an object's type and declared content length from either 39 + // store. 40 + func (dual *Dual) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, error) { 41 + return dual.reader.ReadHeader(id) 42 + } 43 + 44 + // Refresh refreshes both underlying stores and the combined read view. 45 + func (dual *Dual) Refresh() error { 46 + err := dual.object.Refresh() 47 + if err != nil { 48 + return err 49 + } 50 + 51 + err = dual.pack.Refresh() 52 + if err != nil { 53 + return err 54 + } 55 + 56 + return dual.reader.Refresh() 57 + }
+32
object/store/dual/writer_object.go
··· 1 + package dual 2 + 3 + import ( 4 + "io" 5 + 6 + objectid "codeberg.org/lindenii/furgit/object/id" 7 + objecttype "codeberg.org/lindenii/furgit/object/type" 8 + ) 9 + 10 + // WriteReaderContent writes one typed object content stream to the object-wise 11 + // store. 12 + func (dual *Dual) WriteReaderContent(ty objecttype.Type, size int64, src io.Reader) (objectid.ObjectID, error) { 13 + return dual.object.WriteReaderContent(ty, size, src) 14 + } 15 + 16 + // WriteReaderFull writes one full serialized object stream as 17 + // "type size\0content" to the object-wise store. 18 + func (dual *Dual) WriteReaderFull(src io.Reader) (objectid.ObjectID, error) { 19 + return dual.object.WriteReaderFull(src) 20 + } 21 + 22 + // WriteBytesContent writes one typed object content byte slice to the 23 + // object-wise store. 24 + func (dual *Dual) WriteBytesContent(ty objecttype.Type, content []byte) (objectid.ObjectID, error) { 25 + return dual.object.WriteBytesContent(ty, content) 26 + } 27 + 28 + // WriteBytesFull writes one full serialized object byte slice as 29 + // "type size\0content" to the object-wise store. 30 + func (dual *Dual) WriteBytesFull(raw []byte) (objectid.ObjectID, error) { 31 + return dual.object.WriteBytesFull(raw) 32 + }
+12
object/store/dual/writer_pack.go
··· 1 + package dual 2 + 3 + import ( 4 + "io" 5 + 6 + objectstore "codeberg.org/lindenii/furgit/object/store" 7 + ) 8 + 9 + // WritePack ingests one pack stream into the pack-wise store. 10 + func (dual *Dual) WritePack(src io.Reader, opts objectstore.PackWriteOptions) error { 11 + return dual.pack.WritePack(src, opts) 12 + }