this repo has no description
Lexicon resolution (#792)
Adds a `ResolveNSID` method to `identity.BaseDirectory`. I think keeping
DNS TXT resolution stuff together in the same package makes sense, even
if this is a "lexicon" thing, not really an "identity" thing. Could
consider renaming that method to `ResolveLexiconNSID` or similar, to
clarify it is about NSIDs in the context of Lexicons, not "NSIDs in
general" (though they are currently only defined in terms of Lexicons).
Adds a new `ResolvingCatalog` implementation of the `lexicon.Catalog`
interface, which does resolution from the live network and adds to a
local cache. Also `lexicon.ResolveLexiconData` low-level helper which
goes from NSID to Lexicon record data.
+322
-31
+14
atproto/identity/live_test.go
+14
atproto/identity/live_test.go
···
138
138
assert.Error(err)
139
139
assert.ErrorIs(err, ErrHandleResolutionFailed)
140
140
}
141
+
142
+
func TestResolveNSID(t *testing.T) {
143
+
t.Skip("TODO: skipping live network test")
144
+
assert := assert.New(t)
145
+
ctx := context.Background()
146
+
147
+
dir := BaseDirectory{}
148
+
// NOTE: this was a very short temporary NSID when rkey restriction was short
149
+
nsid := syntax.NSID("net.bnewbold.m")
150
+
did, err := dir.ResolveNSID(ctx, nsid)
151
+
152
+
assert.NoError(err)
153
+
assert.Equal(did, syntax.DID("did:plc:nhxcyu4ewwhl5pqil4dotqjo"))
154
+
}
+36
atproto/identity/nsid.go
+36
atproto/identity/nsid.go
···
1
+
package identity
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"fmt"
7
+
"net"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/syntax"
10
+
)
11
+
12
+
var (
13
+
ErrNSIDResolutionFailed = fmt.Errorf("NSID resolution mechanism failed")
14
+
ErrNSIDNotFound = fmt.Errorf("NSID not associated with a DID")
15
+
)
16
+
17
+
// Resolves an NSID to a DID, as used for Lexicon resolution (using "_lexicon" DNS TXT record)
18
+
func (d *BaseDirectory) ResolveNSID(ctx context.Context, nsid syntax.NSID) (syntax.DID, error) {
19
+
20
+
domain := nsid.Authority()
21
+
res, err := d.Resolver.LookupTXT(ctx, "_lexicon."+domain)
22
+
23
+
// check for NXDOMAIN
24
+
var dnsErr *net.DNSError
25
+
if errors.As(err, &dnsErr) {
26
+
if dnsErr.IsNotFound {
27
+
return "", ErrNSIDNotFound
28
+
}
29
+
}
30
+
31
+
if err != nil {
32
+
return "", fmt.Errorf("%w: %w", ErrNSIDResolutionFailed, err)
33
+
}
34
+
35
+
return parseTXTResp(res)
36
+
}
+43
-9
atproto/lexicon/catalog.go
+43
-9
atproto/lexicon/catalog.go
···
1
1
package lexicon
2
2
3
3
import (
4
+
"embed"
4
5
"encoding/json"
5
6
"fmt"
6
7
"io"
···
29
30
}
30
31
}
31
32
33
+
// Returns a scheman definition (`Schema` struct) for a Lexicon reference.
34
+
//
35
+
// A Lexicon ref string is an NSID with an optional #-separated fragment. If the fragment isn't specified, '#main' is used by default.
32
36
func (c *BaseCatalog) Resolve(ref string) (*Schema, error) {
33
37
if ref == "" {
34
38
return nil, fmt.Errorf("tried to resolve empty string name")
···
46
50
47
51
// Inserts a schema loaded from a JSON file in to the catalog.
48
52
func (c *BaseCatalog) AddSchemaFile(sf SchemaFile) error {
53
+
if sf.Lexicon != 1 {
54
+
return fmt.Errorf("unsupported lexicon language version: %d", sf.Lexicon)
55
+
}
49
56
base := sf.ID
50
57
for frag, def := range sf.Defs {
51
58
if len(frag) == 0 || strings.Contains(frag, "#") || strings.Contains(frag, ".") {
···
72
79
return err
73
80
}
74
81
s := Schema{
75
-
ID: name,
76
-
Revision: sf.Revision,
77
-
Def: def.Inner,
82
+
ID: name,
83
+
Def: def.Inner,
78
84
}
79
85
c.schemas[name] = s
80
86
}
81
87
return nil
82
88
}
83
89
90
+
// internal helper for loading JSON files from bytes
91
+
func (c *BaseCatalog) addSchemaFromBytes(b []byte) error {
92
+
var sf SchemaFile
93
+
if err := json.Unmarshal(b, &sf); err != nil {
94
+
return err
95
+
}
96
+
if err := c.AddSchemaFile(sf); err != nil {
97
+
return err
98
+
}
99
+
return nil
100
+
}
101
+
84
102
// Recursively loads all '.json' files from a directory in to the catalog.
85
103
func (c *BaseCatalog) LoadDirectory(dirPath string) error {
86
-
return filepath.WalkDir(dirPath, func(p string, d fs.DirEntry, err error) error {
104
+
walkFunc := func(p string, d fs.DirEntry, err error) error {
87
105
if err != nil {
88
106
return err
89
107
}
···
104
122
if err != nil {
105
123
return err
106
124
}
125
+
return c.addSchemaFromBytes(b)
126
+
}
127
+
return filepath.WalkDir(dirPath, walkFunc)
128
+
}
107
129
108
-
var sf SchemaFile
109
-
if err = json.Unmarshal(b, &sf); err != nil {
130
+
// Recursively loads all '.json' files from an embed.FS
131
+
func (c *BaseCatalog) LoadEmbedFS(efs embed.FS) error {
132
+
walkFunc := func(p string, d fs.DirEntry, err error) error {
133
+
if err != nil {
110
134
return err
111
135
}
112
-
if err = c.AddSchemaFile(sf); err != nil {
136
+
if d.IsDir() {
137
+
return nil
138
+
}
139
+
if !strings.HasSuffix(p, ".json") {
140
+
return nil
141
+
}
142
+
143
+
slog.Debug("loading embedded Lexicon schema file", "path", p)
144
+
b, err := efs.ReadFile(p)
145
+
if err != nil {
113
146
return err
114
147
}
115
-
return nil
116
-
})
148
+
return c.addSchemaFromBytes(b)
149
+
}
150
+
return fs.WalkDir(efs, ".", walkFunc)
117
151
}
+41
atproto/lexicon/catalog_test.go
+41
atproto/lexicon/catalog_test.go
···
1
+
package lexicon
2
+
3
+
import (
4
+
"embed"
5
+
"testing"
6
+
7
+
"github.com/stretchr/testify/assert"
8
+
)
9
+
10
+
//go:embed testdata/catalog
11
+
var embedDir embed.FS
12
+
13
+
func TestEmbedCatalog(t *testing.T) {
14
+
assert := assert.New(t)
15
+
16
+
cat := NewBaseCatalog()
17
+
18
+
err := cat.LoadEmbedFS(embedDir)
19
+
assert.NoError(err)
20
+
21
+
_, err = cat.Resolve("example.lexicon.query")
22
+
assert.NoError(err)
23
+
24
+
_, err = cat.Resolve("example.lexicon.notThere")
25
+
assert.Error(err)
26
+
}
27
+
28
+
func TestDirCatalog(t *testing.T) {
29
+
assert := assert.New(t)
30
+
31
+
cat := NewBaseCatalog()
32
+
33
+
err := cat.LoadDirectory("testdata/catalog")
34
+
assert.NoError(err)
35
+
36
+
_, err = cat.Resolve("example.lexicon.query")
37
+
assert.NoError(err)
38
+
39
+
_, err = cat.Resolve("example.lexicon.notThere")
40
+
assert.Error(err)
41
+
}
+23
-3
atproto/lexicon/cmd/lextool/main.go
+23
-3
atproto/lexicon/cmd/lextool/main.go
···
34
34
Action: runValidateRecord,
35
35
},
36
36
&cli.Command{
37
-
Name: "validate-firehose",
38
-
Usage: "subscribe to a firehose, validate every known record against catalog",
39
-
Action: runValidateFirehose,
37
+
Name: "resolve",
38
+
Usage: "resolves an NSID to a lexicon schema",
39
+
Action: runResolve,
40
40
},
41
41
}
42
42
h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
···
88
88
fmt.Println("success!")
89
89
return nil
90
90
}
91
+
92
+
func runResolve(cctx *cli.Context) error {
93
+
ref := cctx.Args().First()
94
+
if ref == "" {
95
+
return fmt.Errorf("need to provide NSID as an argument")
96
+
}
97
+
98
+
c := lexicon.NewResolvingCatalog()
99
+
schema, err := c.Resolve(ref)
100
+
if err != nil {
101
+
return err
102
+
}
103
+
104
+
out, err := json.MarshalIndent(schema, "", " ")
105
+
if err != nil {
106
+
return err
107
+
}
108
+
fmt.Println(string(out))
109
+
return nil
110
+
}
-15
atproto/lexicon/cmd/lextool/net.go
-15
atproto/lexicon/cmd/lextool/net.go
···
78
78
fmt.Println("success!")
79
79
return nil
80
80
}
81
-
82
-
func runValidateFirehose(cctx *cli.Context) error {
83
-
p := cctx.Args().First()
84
-
if p == "" {
85
-
return fmt.Errorf("need to provide directory path as an argument")
86
-
}
87
-
88
-
cat := lexicon.NewBaseCatalog()
89
-
err := cat.LoadDirectory(p)
90
-
if err != nil {
91
-
return err
92
-
}
93
-
94
-
return fmt.Errorf("UNIMPLEMENTED")
95
-
}
-1
atproto/lexicon/language.go
-1
atproto/lexicon/language.go
+2
-3
atproto/lexicon/lexicon.go
+2
-3
atproto/lexicon/lexicon.go
···
22
22
23
23
// Represents a Lexicon schema definition
24
24
type Schema struct {
25
-
ID string
26
-
Revision *int
27
-
Def any
25
+
ID string
26
+
Def any
28
27
}
29
28
30
29
// Checks Lexicon schema (fetched from the catalog) for the given record, with optional flags tweaking default validation rules.
+89
atproto/lexicon/resolve.go
+89
atproto/lexicon/resolve.go
···
1
+
package lexicon
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"log/slog"
8
+
9
+
"github.com/bluesky-social/indigo/api/agnostic"
10
+
"github.com/bluesky-social/indigo/atproto/data"
11
+
"github.com/bluesky-social/indigo/atproto/identity"
12
+
"github.com/bluesky-social/indigo/atproto/syntax"
13
+
"github.com/bluesky-social/indigo/xrpc"
14
+
)
15
+
16
+
// Low-level routine for resolving an NSID to full Lexicon data record (as stored in a repository).
17
+
//
18
+
// The current implementation uses a naive 'getRepo' fetch to the relevant PDS instance, without validating MST proof chain.
19
+
//
20
+
// Calling code should usually use ResolvingCatalog, which handles basic caching and validation of the Lexicon language itself.
21
+
func ResolveLexiconData(ctx context.Context, dir identity.Directory, nsid syntax.NSID) (map[string]any, error) {
22
+
23
+
record, err := resolveLexiconJSON(ctx, dir, nsid)
24
+
if err != nil {
25
+
return nil, err
26
+
}
27
+
28
+
d, err := data.UnmarshalJSON(*record)
29
+
if err != nil {
30
+
return nil, fmt.Errorf("fetched Lexicon schema record was invalid: %w", err)
31
+
}
32
+
return d, nil
33
+
}
34
+
35
+
// Low-level routine for resolving an NSID to `SchemaFile`.
36
+
//
37
+
// Same as `ResolveLexiconData`, but returns a parsed `SchemaFile` struct.
38
+
func ResolveLexiconSchemaFile(ctx context.Context, dir identity.Directory, nsid syntax.NSID) (*SchemaFile, error) {
39
+
record, err := resolveLexiconJSON(ctx, dir, nsid)
40
+
if err != nil {
41
+
return nil, err
42
+
}
43
+
44
+
var sf SchemaFile
45
+
if err := json.Unmarshal(*record, &sf); err != nil {
46
+
return nil, fmt.Errorf("fetched Lexicon schema record was invalid: %w", err)
47
+
}
48
+
return &sf, nil
49
+
}
50
+
51
+
// internal helper for fetching lexicon record as JSON bytes
52
+
func resolveLexiconJSON(ctx context.Context, dir identity.Directory, nsid syntax.NSID) (*json.RawMessage, error) {
53
+
baseDir := identity.BaseDirectory{}
54
+
did, err := baseDir.ResolveNSID(ctx, nsid)
55
+
if err != nil {
56
+
return nil, err
57
+
}
58
+
slog.Debug("resolved NSID", "nsid", nsid, "did", did)
59
+
60
+
ident, err := dir.LookupDID(ctx, did)
61
+
if err != nil {
62
+
return nil, err
63
+
}
64
+
65
+
aturi := syntax.ATURI(fmt.Sprintf("at://%s/com.atproto.lexicon.schema/%s", did, nsid))
66
+
msg, err := fetchRecordJSON(ctx, *ident, aturi)
67
+
if err != nil {
68
+
return nil, err
69
+
}
70
+
return msg, err
71
+
}
72
+
73
+
func fetchRecordJSON(ctx context.Context, ident identity.Identity, aturi syntax.ATURI) (*json.RawMessage, error) {
74
+
75
+
slog.Debug("fetching record", "did", ident.DID.String(), "collection", aturi.Collection().String(), "rkey", aturi.RecordKey().String())
76
+
xrpcc := xrpc.Client{
77
+
Host: ident.PDSEndpoint(),
78
+
}
79
+
resp, err := agnostic.RepoGetRecord(ctx, &xrpcc, "", aturi.Collection().String(), ident.DID.String(), aturi.RecordKey().String())
80
+
if err != nil {
81
+
return nil, err
82
+
}
83
+
84
+
if nil == resp.Value {
85
+
return nil, fmt.Errorf("empty record in response")
86
+
}
87
+
88
+
return resp.Value, nil
89
+
}
+74
atproto/lexicon/resolving_catalog.go
+74
atproto/lexicon/resolving_catalog.go
···
1
+
package lexicon
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"strings"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/identity"
10
+
"github.com/bluesky-social/indigo/atproto/syntax"
11
+
)
12
+
13
+
// Catalog which supplements an in-memory BaseCatalog with live resolution from the network
14
+
type ResolvingCatalog struct {
15
+
Base BaseCatalog
16
+
Directory identity.Directory
17
+
}
18
+
19
+
func NewResolvingCatalog() ResolvingCatalog {
20
+
return ResolvingCatalog{
21
+
Base: NewBaseCatalog(),
22
+
Directory: identity.DefaultDirectory(),
23
+
}
24
+
}
25
+
26
+
func (rc *ResolvingCatalog) Resolve(ref string) (*Schema, error) {
27
+
// NOTE: not passed through!
28
+
ctx := context.Background()
29
+
30
+
if ref == "" {
31
+
return nil, fmt.Errorf("tried to resolve empty string name")
32
+
}
33
+
34
+
// first try existing catalog
35
+
schema, err := rc.Base.Resolve(ref)
36
+
if nil == err { // no error: found a hit
37
+
return schema, nil
38
+
}
39
+
40
+
// split any ref from the end '#'
41
+
parts := strings.SplitN(ref, "#", 2)
42
+
nsid, err := syntax.ParseNSID(parts[0])
43
+
if err != nil {
44
+
return nil, err
45
+
}
46
+
47
+
record, err := ResolveLexiconData(ctx, rc.Directory, nsid)
48
+
if err != nil {
49
+
return nil, err
50
+
}
51
+
52
+
recordJSON, err := json.Marshal(record)
53
+
if err != nil {
54
+
return nil, err
55
+
}
56
+
57
+
var sf SchemaFile
58
+
if err = json.Unmarshal(recordJSON, &sf); err != nil {
59
+
return nil, err
60
+
}
61
+
62
+
if sf.Lexicon != 1 {
63
+
return nil, fmt.Errorf("unsupported lexicon language version: %d", sf.Lexicon)
64
+
}
65
+
if sf.ID != nsid.String() {
66
+
return nil, fmt.Errorf("lexicon ID does not match NSID: %s != %s", sf.ID, nsid)
67
+
}
68
+
if err = rc.Base.AddSchemaFile(sf); err != nil {
69
+
return nil, err
70
+
}
71
+
72
+
// re-resolving from the raw ref ensures that fragments are handled
73
+
return rc.Base.Resolve(ref)
74
+
}