this repo has no description
Add debug command to compare the same repo across two hosts (#366)
+153
-1
+153
cmd/gosky/debug.go
+153
cmd/gosky/debug.go
···
11
11
"os/signal"
12
12
"strconv"
13
13
"strings"
14
+
"sync"
14
15
"syscall"
16
+
"time"
15
17
16
18
"github.com/bluesky-social/indigo/api/atproto"
17
19
comatproto "github.com/bluesky-social/indigo/api/atproto"
···
28
30
29
31
"github.com/gorilla/websocket"
30
32
"github.com/ipfs/go-cid"
33
+
"github.com/ipfs/go-libipfs/blocks"
31
34
"github.com/ipld/go-car/v2"
32
35
cli "github.com/urfave/cli/v2"
33
36
)
···
42
45
debugFeedViewCmd,
43
46
compareStreamsCmd,
44
47
debugGetRepoCmd,
48
+
debugCompareReposCmd,
45
49
},
46
50
}
47
51
···
789
793
return nil
790
794
},
791
795
}
796
+
797
+
var debugCompareReposCmd = &cli.Command{
798
+
Name: "compare-repos",
799
+
Flags: []cli.Flag{
800
+
&cli.StringFlag{
801
+
Name: "host-1",
802
+
Usage: "method, hostname, and port of PDS instance",
803
+
Value: "https://bsky.social",
804
+
},
805
+
&cli.StringFlag{
806
+
Name: "host-2",
807
+
Usage: "method, hostname, and port of PDS instance",
808
+
Value: "https://bgs.bsky.social",
809
+
},
810
+
},
811
+
ArgsUsage: `<did>`,
812
+
Action: func(cctx *cli.Context) error {
813
+
ctx := cctx.Context
814
+
815
+
wg := sync.WaitGroup{}
816
+
wg.Add(2)
817
+
818
+
xrpc1 := xrpc.Client{
819
+
Host: cctx.String("host-1"),
820
+
Client: &http.Client{
821
+
Timeout: 15 * time.Minute,
822
+
},
823
+
}
824
+
825
+
xrpc2 := xrpc.Client{
826
+
Host: cctx.String("host-2"),
827
+
Client: &http.Client{
828
+
Timeout: 15 * time.Minute,
829
+
},
830
+
}
831
+
832
+
var rep1 *repo.Repo
833
+
go func() {
834
+
defer wg.Done()
835
+
logger := log.With("host", cctx.String("host-1"))
836
+
repo1bytes, err := comatproto.SyncGetRepo(ctx, &xrpc1, cctx.Args().First(), "")
837
+
if err != nil {
838
+
logger.Fatalf("getting repo: %s", err)
839
+
return
840
+
}
841
+
842
+
rep1, err = repo.ReadRepoFromCar(ctx, bytes.NewReader(repo1bytes))
843
+
if err != nil {
844
+
logger.Fatalf("reading repo: %s", err)
845
+
return
846
+
}
847
+
}()
848
+
849
+
var rep2 *repo.Repo
850
+
go func() {
851
+
defer wg.Done()
852
+
logger := log.With("host", cctx.String("host-2"))
853
+
repo2bytes, err := comatproto.SyncGetRepo(ctx, &xrpc2, cctx.Args().First(), "")
854
+
if err != nil {
855
+
logger.Fatalf("getting repo: %s", err)
856
+
return
857
+
}
858
+
859
+
rep2, err = repo.ReadRepoFromCar(ctx, bytes.NewReader(repo2bytes))
860
+
if err != nil {
861
+
logger.Fatalf("reading repo: %s", err)
862
+
return
863
+
}
864
+
}()
865
+
866
+
wg.Wait()
867
+
868
+
cids1 := []cid.Cid{}
869
+
blocks1 := []blocks.Block{}
870
+
871
+
fmt.Println("Host 1 Results")
872
+
fmt.Println("Rev: ", rep1.SignedCommit().Rev)
873
+
var count int
874
+
if err := rep1.ForEach(ctx, "", func(k string, v cid.Cid) error {
875
+
cids1 = append(cids1, v)
876
+
rec, err := rep1.Blockstore().Get(ctx, v)
877
+
if err != nil {
878
+
return fmt.Errorf("getting record %q: %w", k, err)
879
+
}
880
+
blocks1 = append(blocks1, rec)
881
+
882
+
count++
883
+
_ = rec
884
+
return nil
885
+
}); err != nil {
886
+
return err
887
+
}
888
+
fmt.Printf("scanned %d records\n", count)
889
+
890
+
cids2 := []cid.Cid{}
891
+
blocks2 := []blocks.Block{}
892
+
893
+
fmt.Println("\nHost 2 Results")
894
+
fmt.Println("Rev: ", rep2.SignedCommit().Rev)
895
+
count = 0
896
+
if err := rep2.ForEach(ctx, "", func(k string, v cid.Cid) error {
897
+
cids2 = append(cids2, v)
898
+
rec, err := rep2.Blockstore().Get(ctx, v)
899
+
if err != nil {
900
+
return fmt.Errorf("getting record %q: %w", k, err)
901
+
}
902
+
blocks2 = append(blocks2, rec)
903
+
904
+
count++
905
+
_ = rec
906
+
return nil
907
+
}); err != nil {
908
+
return err
909
+
}
910
+
fmt.Printf("scanned %d records\n", count)
911
+
912
+
fmt.Println("\nComparing CIDs")
913
+
hasBadCid := false
914
+
for i, c1 := range cids1 {
915
+
if c1 != cids2[i] {
916
+
fmt.Printf("CID mismatch at index %d: %s != %s\n", i, c1, cids2[i])
917
+
hasBadCid = true
918
+
}
919
+
}
920
+
921
+
if !hasBadCid {
922
+
fmt.Println("All CIDs match!")
923
+
}
924
+
925
+
fmt.Println("Comparing blocks")
926
+
hasBadBlock := false
927
+
for i, b1 := range blocks1 {
928
+
if !bytes.Equal(b1.RawData(), blocks2[i].RawData()) {
929
+
fmt.Printf("Block mismatch at index %d Host 1 Cid (%s) Host 2 Cid (%s)\n", i, b1.Cid().String(), blocks2[i].Cid().String())
930
+
hasBadBlock = true
931
+
}
932
+
}
933
+
934
+
if !hasBadBlock {
935
+
fmt.Println("All blocks match!")
936
+
}
937
+
938
+
if hasBadBlock || hasBadCid {
939
+
return fmt.Errorf("mismatched blocks or cids")
940
+
}
941
+
942
+
return nil
943
+
},
944
+
}