this repo has no description
goat: repo mst command for displaying MST structure (#885)
+217
-33
+161
-6
cmd/goat/repo.go
+161
-6
cmd/goat/repo.go
···
4
4
"bytes"
5
5
"context"
6
6
"encoding/json"
7
+
"errors"
7
8
"fmt"
8
9
"os"
9
10
"path/filepath"
11
+
"strings"
10
12
"time"
11
13
12
14
comatproto "github.com/bluesky-social/indigo/api/atproto"
13
15
"github.com/bluesky-social/indigo/atproto/data"
14
16
"github.com/bluesky-social/indigo/atproto/syntax"
17
+
"github.com/bluesky-social/indigo/mst"
15
18
"github.com/bluesky-social/indigo/repo"
19
+
"github.com/bluesky-social/indigo/util"
16
20
"github.com/bluesky-social/indigo/xrpc"
17
-
18
21
"github.com/ipfs/go-cid"
22
+
cbor "github.com/ipfs/go-ipld-cbor"
23
+
ipld "github.com/ipfs/go-ipld-format"
19
24
"github.com/urfave/cli/v2"
25
+
"github.com/xlab/treeprint"
20
26
)
21
27
22
28
var cmdRepo = &cli.Command{
···
59
65
Action: runRepoInspect,
60
66
},
61
67
&cli.Command{
68
+
Name: "mst",
69
+
Usage: "show repo MST structure",
70
+
ArgsUsage: `<car-file>`,
71
+
Flags: []cli.Flag{
72
+
&cli.BoolFlag{
73
+
Name: "full-cid",
74
+
Aliases: []string{"f"},
75
+
Usage: "display full CIDs",
76
+
},
77
+
&cli.StringFlag{
78
+
Name: "root",
79
+
Aliases: []string{"r"},
80
+
Usage: "CID of root block",
81
+
},
82
+
},
83
+
Action: runRepoMST,
84
+
},
85
+
&cli.Command{
62
86
Name: "unpack",
63
87
Usage: "extract records from CAR file as directory of JSON files",
64
88
ArgsUsage: `<car-file>`,
···
99
123
now := time.Now().Format("20060102150405")
100
124
carPath = fmt.Sprintf("%s.%s.car", username, now)
101
125
}
102
-
// NOTE: there is a race condition, but nice to give a friendly error earlier before downloading
103
-
if _, err := os.Stat(carPath); err == nil {
104
-
return fmt.Errorf("file already exists: %s", carPath)
126
+
output, err := getFileOrStdout(carPath)
127
+
if err != nil {
128
+
if errors.Is(err, os.ErrExist) {
129
+
return fmt.Errorf("file already exists: %s", carPath)
130
+
}
131
+
return err
105
132
}
106
-
fmt.Printf("downloading from %s to: %s\n", xrpcc.Host, carPath)
133
+
defer output.Close()
134
+
if carPath != stdIOPath {
135
+
fmt.Printf("downloading from %s to: %s\n", xrpcc.Host, carPath)
136
+
}
107
137
repoBytes, err := comatproto.SyncGetRepo(ctx, &xrpcc, ident.DID.String(), "")
108
138
if err != nil {
109
139
return err
110
140
}
111
-
return os.WriteFile(carPath, repoBytes, 0666)
141
+
if _, err := output.Write(repoBytes); err != nil {
142
+
return err
143
+
}
144
+
return nil
112
145
}
113
146
114
147
func runRepoImport(cctx *cli.Context) error {
···
192
225
// TODO: Signature?
193
226
194
227
return nil
228
+
}
229
+
230
+
func runRepoMST(cctx *cli.Context) error {
231
+
ctx := context.Background()
232
+
opts := repoMSTOptions{
233
+
carPath: cctx.Args().First(),
234
+
fullCID: cctx.Bool("full-cid"),
235
+
root: cctx.String("root"),
236
+
}
237
+
// read from file or stdin
238
+
if opts.carPath == "" {
239
+
return fmt.Errorf("need to provide path to CAR file as argument")
240
+
}
241
+
inputCAR, err := getFileOrStdin(opts.carPath)
242
+
if err != nil {
243
+
return err
244
+
}
245
+
// read repository tree in to memory
246
+
r, err := repo.ReadRepoFromCar(ctx, inputCAR)
247
+
if err != nil {
248
+
return err
249
+
}
250
+
cst := util.CborStore(r.Blockstore())
251
+
// determine which root cid to use, defaulting to repo data root
252
+
rootCID := r.DataCid()
253
+
if opts.root != "" {
254
+
optsRootCID, err := cid.Decode(opts.root)
255
+
if err != nil {
256
+
return err
257
+
}
258
+
rootCID = optsRootCID
259
+
}
260
+
// start walking mst
261
+
exists, err := nodeExists(ctx, cst, rootCID)
262
+
if err != nil {
263
+
return err
264
+
}
265
+
tree := treeprint.NewWithRoot(displayCID(&rootCID, exists, opts))
266
+
if exists {
267
+
if err := walkMST(ctx, cst, rootCID, tree, opts); err != nil {
268
+
return err
269
+
}
270
+
}
271
+
// print tree
272
+
fmt.Println(tree.String())
273
+
return nil
274
+
}
275
+
276
+
func walkMST(ctx context.Context, cst *cbor.BasicIpldStore, cid cid.Cid, tree treeprint.Tree, opts repoMSTOptions) error {
277
+
var node mst.NodeData
278
+
if err := cst.Get(ctx, cid, &node); err != nil {
279
+
return err
280
+
}
281
+
if node.Left != nil {
282
+
exists, err := nodeExists(ctx, cst, *node.Left)
283
+
if err != nil {
284
+
return err
285
+
}
286
+
subtree := tree.AddBranch(displayCID(node.Left, exists, opts))
287
+
if exists {
288
+
if err := walkMST(ctx, cst, *node.Left, subtree, opts); err != nil {
289
+
return err
290
+
}
291
+
}
292
+
}
293
+
for _, entry := range node.Entries {
294
+
exists, err := nodeExists(ctx, cst, entry.Val)
295
+
if err != nil {
296
+
return err
297
+
}
298
+
tree.AddNode(displayEntryVal(&entry, exists, opts))
299
+
if entry.Tree != nil {
300
+
exists, err := nodeExists(ctx, cst, *entry.Tree)
301
+
if err != nil {
302
+
return err
303
+
}
304
+
subtree := tree.AddBranch(displayCID(entry.Tree, exists, opts))
305
+
if exists {
306
+
if err := walkMST(ctx, cst, *entry.Tree, subtree, opts); err != nil {
307
+
return err
308
+
}
309
+
}
310
+
}
311
+
}
312
+
return nil
313
+
}
314
+
315
+
func displayEntryVal(entry *mst.TreeEntry, exists bool, opts repoMSTOptions) string {
316
+
key := string(entry.KeySuffix)
317
+
divider := " "
318
+
if opts.fullCID {
319
+
divider = "\n"
320
+
}
321
+
return strings.Repeat("∙", int(entry.PrefixLen)) + key + divider + displayCID(&entry.Val, exists, opts)
322
+
}
323
+
324
+
func displayCID(cid *cid.Cid, exists bool, opts repoMSTOptions) string {
325
+
cidDisplay := cid.String()
326
+
if !opts.fullCID {
327
+
cidDisplay = "…" + string(cidDisplay[len(cidDisplay)-7:])
328
+
}
329
+
connector := "─◉"
330
+
if !exists {
331
+
connector = "─◌"
332
+
}
333
+
return "[" + cidDisplay + "]" + connector
334
+
}
335
+
336
+
type repoMSTOptions struct {
337
+
carPath string
338
+
fullCID bool
339
+
root string
340
+
}
341
+
342
+
func nodeExists(ctx context.Context, cst *cbor.BasicIpldStore, cid cid.Cid) (bool, error) {
343
+
if _, err := cst.Blocks.Get(ctx, cid); err != nil {
344
+
if errors.Is(err, ipld.ErrNotFound{}) {
345
+
return false, nil
346
+
}
347
+
return false, err
348
+
}
349
+
return true, nil
195
350
}
196
351
197
352
func runRepoUnpack(cctx *cli.Context) error {
+26
cmd/goat/util.go
+26
cmd/goat/util.go
···
2
2
3
3
import (
4
4
"context"
5
+
"io"
6
+
"os"
5
7
6
8
"github.com/bluesky-social/indigo/atproto/identity"
7
9
"github.com/bluesky-social/indigo/atproto/syntax"
···
16
18
dir := identity.DefaultDirectory()
17
19
return dir.Lookup(ctx, *id)
18
20
}
21
+
22
+
const stdIOPath = "-"
23
+
24
+
func getFileOrStdin(path string) (io.Reader, error) {
25
+
if path == stdIOPath {
26
+
return os.Stdin, nil
27
+
}
28
+
file, err := os.Open(path)
29
+
if err != nil {
30
+
return nil, err
31
+
}
32
+
return file, nil
33
+
}
34
+
35
+
func getFileOrStdout(path string) (io.WriteCloser, error) {
36
+
if path == stdIOPath {
37
+
return os.Stdout, nil
38
+
}
39
+
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
40
+
if err != nil {
41
+
return nil, err
42
+
}
43
+
return file, nil
44
+
}
+3
-2
go.mod
+3
-2
go.mod
···
31
31
github.com/ipfs/go-ipld-cbor v0.1.0
32
32
github.com/ipfs/go-ipld-format v0.6.0
33
33
github.com/ipfs/go-libipfs v0.7.0
34
+
github.com/ipfs/go-log/v2 v2.5.1
34
35
github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4
35
36
github.com/ipld/go-car/v2 v2.13.1
36
37
github.com/jackc/pgx/v5 v5.5.0
···
55
56
github.com/urfave/cli/v2 v2.25.7
56
57
github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e
57
58
github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6
59
+
github.com/xlab/treeprint v1.2.0
58
60
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b
59
61
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1
60
62
go.opentelemetry.io/otel v1.21.0
···
64
66
go.opentelemetry.io/otel/sdk v1.21.0
65
67
go.opentelemetry.io/otel/trace v1.21.0
66
68
go.uber.org/automaxprocs v1.5.3
69
+
go.uber.org/zap v1.26.0
67
70
golang.org/x/crypto v0.21.0
68
71
golang.org/x/sync v0.7.0
69
72
golang.org/x/text v0.14.0
···
89
92
github.com/golang/snappy v0.0.4 // indirect
90
93
github.com/hashicorp/golang-lru v1.0.2 // indirect
91
94
github.com/ipfs/go-log v1.0.5 // indirect
92
-
github.com/ipfs/go-log/v2 v2.5.1 // indirect
93
95
github.com/jackc/puddle/v2 v2.2.1 // indirect
94
96
github.com/klauspost/compress v1.17.3 // indirect
95
97
github.com/kr/pretty v0.3.1 // indirect
···
103
105
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
104
106
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
105
107
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
106
-
go.uber.org/zap v1.26.0 // indirect
107
108
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
108
109
)
109
110
+2
go.sum
+2
go.sum
···
651
651
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
652
652
github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6 h1:yJ9/LwIGIk/c0CdoavpC9RNSGSruIspSZtxG3Nnldic=
653
653
github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6/go.mod h1:39U9RRVr4CKbXpXYopWn+FSH5s+vWu6+RmguSPWAq5s=
654
+
github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
655
+
github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
654
656
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
655
657
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
656
658
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+11
-11
mst/cbor_gen.go
+11
-11
mst/cbor_gen.go
···
18
18
var _ = math.E
19
19
var _ = sort.Sort
20
20
21
-
func (t *nodeData) MarshalCBOR(w io.Writer) error {
21
+
func (t *NodeData) MarshalCBOR(w io.Writer) error {
22
22
if t == nil {
23
23
_, err := w.Write(cbg.CborNull)
24
24
return err
···
30
30
return err
31
31
}
32
32
33
-
// t.Entries ([]mst.treeEntry) (slice)
33
+
// t.Entries ([]mst.TreeEntry) (slice)
34
34
if len("e") > 1000000 {
35
35
return xerrors.Errorf("Value in field \"e\" was too long")
36
36
}
···
81
81
return nil
82
82
}
83
83
84
-
func (t *nodeData) UnmarshalCBOR(r io.Reader) (err error) {
85
-
*t = nodeData{}
84
+
func (t *NodeData) UnmarshalCBOR(r io.Reader) (err error) {
85
+
*t = NodeData{}
86
86
87
87
cr := cbg.NewCborReader(r)
88
88
···
101
101
}
102
102
103
103
if extra > cbg.MaxLength {
104
-
return fmt.Errorf("nodeData: map struct too large (%d)", extra)
104
+
return fmt.Errorf("NodeData: map struct too large (%d)", extra)
105
105
}
106
106
107
107
n := extra
···
122
122
}
123
123
124
124
switch string(nameBuf[:nameLen]) {
125
-
// t.Entries ([]mst.treeEntry) (slice)
125
+
// t.Entries ([]mst.TreeEntry) (slice)
126
126
case "e":
127
127
128
128
maj, extra, err = cr.ReadHeader()
···
139
139
}
140
140
141
141
if extra > 0 {
142
-
t.Entries = make([]treeEntry, extra)
142
+
t.Entries = make([]TreeEntry, extra)
143
143
}
144
144
145
145
for i := 0; i < int(extra); i++ {
···
195
195
196
196
return nil
197
197
}
198
-
func (t *treeEntry) MarshalCBOR(w io.Writer) error {
198
+
func (t *TreeEntry) MarshalCBOR(w io.Writer) error {
199
199
if t == nil {
200
200
_, err := w.Write(cbg.CborNull)
201
201
return err
···
294
294
return nil
295
295
}
296
296
297
-
func (t *treeEntry) UnmarshalCBOR(r io.Reader) (err error) {
298
-
*t = treeEntry{}
297
+
func (t *TreeEntry) UnmarshalCBOR(r io.Reader) (err error) {
298
+
*t = TreeEntry{}
299
299
300
300
cr := cbg.NewCborReader(r)
301
301
···
314
314
}
315
315
316
316
if extra > cbg.MaxLength {
317
-
return fmt.Errorf("treeEntry: map struct too large (%d)", extra)
317
+
return fmt.Errorf("TreeEntry: map struct too large (%d)", extra)
318
318
}
319
319
320
320
n := extra
+8
-8
mst/mst.go
+8
-8
mst/mst.go
···
105
105
// the CBOR codec.
106
106
func CBORTypes() []reflect.Type {
107
107
return []reflect.Type{
108
-
reflect.TypeOf(nodeData{}),
109
-
reflect.TypeOf(treeEntry{}),
108
+
reflect.TypeOf(NodeData{}),
109
+
reflect.TypeOf(TreeEntry{}),
110
110
}
111
111
}
112
112
113
113
// MST tree node as gets serialized to CBOR. Note that the CBOR fields are all
114
114
// single-character.
115
-
type nodeData struct {
115
+
type NodeData struct {
116
116
Left *cid.Cid `cborgen:"l"` // [nullable] pointer to lower-level subtree to the "left" of this path/key
117
-
Entries []treeEntry `cborgen:"e"` // ordered list of entries at this node
117
+
Entries []TreeEntry `cborgen:"e"` // ordered list of entries at this node
118
118
}
119
119
120
-
// treeEntry are elements of nodeData's Entries.
121
-
type treeEntry struct {
120
+
// TreeEntry are elements of NodeData's Entries.
121
+
type TreeEntry struct {
122
122
PrefixLen int64 `cborgen:"p"` // count of characters shared with previous path/key in tree
123
123
KeySuffix []byte `cborgen:"k"` // remaining part of path/key (appended to "previous key")
124
124
Val cid.Cid `cborgen:"v"` // CID pointer at this path/key
···
189
189
// otherwise this is a virtual/pointer struct and we need to hydrate from
190
190
// blockstore before returning entries
191
191
if mst.pointer != cid.Undef {
192
-
var nd nodeData
192
+
var nd NodeData
193
193
if err := mst.cst.Get(ctx, mst.pointer, &nd); err != nil {
194
194
return nil, err
195
195
}
···
210
210
}
211
211
212
212
// golang-specific helper that calls in to deserializeNodeData
213
-
func entriesFromNodeData(ctx context.Context, nd *nodeData, cst cbor.IpldStore) ([]nodeEntry, error) {
213
+
func entriesFromNodeData(ctx context.Context, nd *NodeData, cst cbor.IpldStore) ([]nodeEntry, error) {
214
214
layer := -1
215
215
if len(nd.Entries) > 0 {
216
216
// NOTE(bnewbold): can compute the layer on the first KeySuffix, because for the first entry that field is a complete key
+2
-2
mst/mst_interop_test.go
+2
-2
mst/mst_interop_test.go
+4
-4
mst/mst_util.go
+4
-4
mst/mst_util.go
···
66
66
}
67
67
68
68
// Typescript: deserializeNodeData(storage, data, layer)
69
-
func deserializeNodeData(ctx context.Context, cst cbor.IpldStore, nd *nodeData, layer int) ([]nodeEntry, error) {
69
+
func deserializeNodeData(ctx context.Context, cst cbor.IpldStore, nd *NodeData, layer int) ([]nodeEntry, error) {
70
70
entries := []nodeEntry{}
71
71
if nd.Left != nil {
72
72
// Note: like Typescript, this is actually a lazy load
···
111
111
}
112
112
113
113
// Typescript: serializeNodeData(entries) -> NodeData
114
-
func serializeNodeData(entries []nodeEntry) (*nodeData, error) {
115
-
var data nodeData
114
+
func serializeNodeData(entries []nodeEntry) (*NodeData, error) {
115
+
var data NodeData
116
116
117
117
i := 0
118
118
if len(entries) > 0 && entries[0].isTree() {
···
157
157
}
158
158
159
159
prefixLen := countPrefixLen(lastKey, leaf.Key)
160
-
data.Entries = append(data.Entries, treeEntry{
160
+
data.Entries = append(data.Entries, TreeEntry{
161
161
PrefixLen: int64(prefixLen),
162
162
KeySuffix: []byte(leaf.Key)[prefixLen:],
163
163
Val: leaf.Val,