this repo has no description
optimize block_ref creation (#203)
The GORM create helpers dont have any way to avoid 'returning' the ID
values of created records, and that was causing a noticeable performance
hit in this critical section.
authored by
Whyrusleeping
and committed by
GitHub
2ff44d97
14a90751
+46
-2
+46
-2
carstore/bs.go
+46
-2
carstore/bs.go
···
9
9
"io"
10
10
"os"
11
11
"path/filepath"
12
+
"strings"
12
13
"sync"
13
14
"time"
14
15
···
636
637
// TODO: there should be a way to create the shard and block_refs that
637
638
// reference it in the same query, would save a lot of time
638
639
if err := ds.cs.meta.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
639
-
if err := tx.Create(&shard).Error; err != nil {
640
+
if err := tx.WithContext(ctx).Create(&shard).Error; err != nil {
640
641
return err
641
642
}
642
643
ds.cs.putLastShardCache(ds.user, &shard)
···
645
646
ref["shard"] = shard.ID
646
647
}
647
648
648
-
if err := tx.Table("block_refs").CreateInBatches(brefs, 100).Error; err != nil {
649
+
if err := createBlockRefs(ctx, tx, brefs); err != nil {
649
650
return err
650
651
}
651
652
···
655
656
}
656
657
657
658
return buf.Bytes(), nil
659
+
}
660
+
661
+
func createBlockRefs(ctx context.Context, tx *gorm.DB, brefs []map[string]any) error {
662
+
ctx, span := otel.Tracer("carstore").Start(ctx, "createBlockRefs")
663
+
defer span.End()
664
+
665
+
if err := createInBatches(ctx, tx, brefs, 100); err != nil {
666
+
return err
667
+
}
668
+
669
+
return nil
670
+
}
671
+
672
+
func generateInsertQuery(data []map[string]any) (string, []any) {
673
+
placeholders := strings.Repeat("(?, ?, ?),", len(data))
674
+
placeholders = placeholders[:len(placeholders)-1] // trim trailing comma
675
+
676
+
query := "INSERT INTO block_refs (cid, offset, shard) VALUES " + placeholders
677
+
678
+
values := make([]any, 0, 3*len(data))
679
+
for _, entry := range data {
680
+
values = append(values, entry["cid"], entry["offset"], entry["shard"])
681
+
}
682
+
683
+
return query, values
684
+
}
685
+
686
+
// Function to create in batches
687
+
func createInBatches(ctx context.Context, tx *gorm.DB, data []map[string]any, batchSize int) error {
688
+
for i := 0; i < len(data); i += batchSize {
689
+
end := i + batchSize
690
+
if end > len(data) {
691
+
end = len(data)
692
+
}
693
+
694
+
batch := data[i:end]
695
+
query, values := generateInsertQuery(batch)
696
+
697
+
if err := tx.WithContext(ctx).Exec(query, values...).Error; err != nil {
698
+
return err
699
+
}
700
+
}
701
+
return nil
658
702
}
659
703
660
704
func (ds *DeltaSession) CloseAsRebase(ctx context.Context, root cid.Cid) error {