this repo has no description
allow carstore to use multiple directories, round robin style (#812)
authored by
Whyrusleeping
and committed by
GitHub
05b87518
0fc4c7e2
+55
-28
+20
-13
carstore/bs.go
+20
-13
carstore/bs.go
···
62
62
}
63
63
64
64
type FileCarStore struct {
65
-
meta *CarStoreGormMeta
66
-
rootDir string
65
+
meta *CarStoreGormMeta
66
+
rootDirs []string
67
67
68
68
lscLk sync.Mutex
69
69
lastShardCache map[models.Uid]*CarShard
70
70
}
71
71
72
-
func NewCarStore(meta *gorm.DB, root string) (CarStore, error) {
73
-
if _, err := os.Stat(root); err != nil {
74
-
if !os.IsNotExist(err) {
75
-
return nil, err
76
-
}
72
+
func NewCarStore(meta *gorm.DB, roots []string) (CarStore, error) {
73
+
for _, root := range roots {
74
+
if _, err := os.Stat(root); err != nil {
75
+
if !os.IsNotExist(err) {
76
+
return nil, err
77
+
}
77
78
78
-
if err := os.Mkdir(root, 0775); err != nil {
79
-
return nil, err
79
+
if err := os.Mkdir(root, 0775); err != nil {
80
+
return nil, err
81
+
}
80
82
}
81
83
}
82
84
if err := meta.AutoMigrate(&CarShard{}, &blockRef{}); err != nil {
···
88
90
89
91
return &FileCarStore{
90
92
meta: &CarStoreGormMeta{meta: meta},
91
-
rootDir: root,
93
+
rootDirs: roots,
92
94
lastShardCache: make(map[models.Uid]*CarShard),
93
95
}, nil
94
96
}
···
541
543
func fnameForShard(user models.Uid, seq int) string {
542
544
return fmt.Sprintf("sh-%d-%d", user, seq)
543
545
}
546
+
547
+
func (cs *FileCarStore) dirForUser(user models.Uid) string {
548
+
return cs.rootDirs[int(user)%len(cs.rootDirs)]
549
+
}
550
+
544
551
func (cs *FileCarStore) openNewShardFile(ctx context.Context, user models.Uid, seq int) (*os.File, string, error) {
545
552
// TODO: some overwrite protections
546
-
fname := filepath.Join(cs.rootDir, fnameForShard(user, seq))
553
+
fname := filepath.Join(cs.dirForUser(user), fnameForShard(user, seq))
547
554
fi, err := os.Create(fname)
548
555
if err != nil {
549
556
return nil, "", err
···
557
564
defer span.End()
558
565
559
566
// TODO: some overwrite protections
560
-
fname := filepath.Join(cs.rootDir, fnameForShard(user, seq))
567
+
fname := filepath.Join(cs.dirForUser(user), fnameForShard(user, seq))
561
568
if err := os.WriteFile(fname, data, 0664); err != nil {
562
569
return "", err
563
570
}
···
982
989
// TODO: some overwrite protections
983
990
// NOTE CreateTemp is used for creating a non-colliding file, but we keep it and don't delete it so don't think of it as "temporary".
984
991
// This creates "sh-%d-%d%s" with some random stuff in the last position
985
-
fi, err := os.CreateTemp(cs.rootDir, fnameForShard(user, seq))
992
+
fi, err := os.CreateTemp(cs.dirForUser(user), fnameForShard(user, seq))
986
993
if err != nil {
987
994
return nil, "", err
988
995
}
+8
-3
carstore/repo_test.go
+8
-3
carstore/repo_test.go
···
30
30
return nil, nil, err
31
31
}
32
32
33
-
sharddir := filepath.Join(tempdir, "shards")
34
-
if err := os.MkdirAll(sharddir, 0775); err != nil {
33
+
sharddir1 := filepath.Join(tempdir, "shards1")
34
+
if err := os.MkdirAll(sharddir1, 0775); err != nil {
35
+
return nil, nil, err
36
+
}
37
+
38
+
sharddir2 := filepath.Join(tempdir, "shards2")
39
+
if err := os.MkdirAll(sharddir2, 0775); err != nil {
35
40
return nil, nil, err
36
41
}
37
42
···
45
50
return nil, nil, err
46
51
}
47
52
48
-
cs, err := NewCarStore(db, sharddir)
53
+
cs, err := NewCarStore(db, []string{sharddir1, sharddir2})
49
54
if err != nil {
50
55
return nil, nil, err
51
56
}
+17
-2
cmd/bigsky/main.go
+17
-2
cmd/bigsky/main.go
···
200
200
EnvVars: []string{"RELAY_NUM_COMPACTION_WORKERS"},
201
201
Value: 2,
202
202
},
203
+
&cli.StringSliceFlag{
204
+
Name: "carstore-shard-dirs",
205
+
Usage: "specify list of shard directories for carstore storage, overrides default storage within datadir",
206
+
EnvVars: []string{"RELAY_CARSTORE_SHARD_DIRS"},
207
+
},
203
208
}
204
209
205
210
app.Action = runBigsky
···
312
317
}
313
318
}
314
319
315
-
os.MkdirAll(filepath.Dir(csdir), os.ModePerm)
316
-
cstore, err := carstore.NewCarStore(csdb, csdir)
320
+
csdirs := []string{csdir}
321
+
if paramDirs := cctx.StringSlice("carstore-shard-dirs"); len(paramDirs) > 0 {
322
+
csdirs = paramDirs
323
+
}
324
+
325
+
for _, csd := range csdirs {
326
+
if err := os.MkdirAll(filepath.Dir(csd), os.ModePerm); err != nil {
327
+
return err
328
+
}
329
+
}
330
+
331
+
cstore, err := carstore.NewCarStore(csdb, csdirs)
317
332
if err != nil {
318
333
return err
319
334
}
+1
-1
cmd/laputa/main.go
+1
-1
cmd/laputa/main.go
+1
-1
cmd/supercollider/main.go
+1
-1
cmd/supercollider/main.go
+1
-1
events/dbpersist_test.go
+1
-1
events/dbpersist_test.go
+1
-1
indexer/posts_test.go
+1
-1
indexer/posts_test.go
+1
-1
pds/handlers_test.go
+1
-1
pds/handlers_test.go
+1
-1
repomgr/bench_test.go
+1
-1
repomgr/bench_test.go
+2
-2
repomgr/ingest_test.go
+2
-2
repomgr/ingest_test.go
···
50
50
t.Fatal(err)
51
51
}
52
52
53
-
cs, err := carstore.NewCarStore(cardb, cspath)
53
+
cs, err := carstore.NewCarStore(cardb, []string{cspath})
54
54
if err != nil {
55
55
t.Fatal(err)
56
56
}
···
80
80
t.Fatal(err)
81
81
}
82
82
83
-
cs, err := carstore.NewCarStore(cardb, cspath)
83
+
cs, err := carstore.NewCarStore(cardb, []string{cspath})
84
84
if err != nil {
85
85
t.Fatal(err)
86
86
}
+2
-2
testing/utils.go
+2
-2
testing/utils.go
···
117
117
return nil, err
118
118
}
119
119
120
-
cs, err := carstore.NewCarStore(cardb, cspath)
120
+
cs, err := carstore.NewCarStore(cardb, []string{cspath})
121
121
if err != nil {
122
122
return nil, err
123
123
}
···
550
550
return nil, err
551
551
}
552
552
553
-
cs, err := carstore.NewCarStore(cardb, cspath)
553
+
cs, err := carstore.NewCarStore(cardb, []string{cspath})
554
554
if err != nil {
555
555
return nil, err
556
556
}