forked from
stavola.xyz/mlf
A human-friendly DSL for ATProto Lexicons
Fetch rework
+160
-82
+59
-12
mlf-cli/src/fetch.rs
+59
-12
mlf-cli/src/fetch.rs
···
231
231
let cache_file = mlf_dir.join(".lexicon-cache.toml");
232
232
let mut cache = LexiconCache::load(&cache_file)?;
233
233
234
-
// Check if already cached
235
-
if cache.lexicons.contains_key(nsid) {
234
+
// Validate NSID format: must be specific (3+ segments) or use wildcard
235
+
validate_nsid_format(nsid)?;
236
+
237
+
// Check if it's a wildcard pattern
238
+
let is_wildcard = nsid.ends_with(".*");
239
+
let nsid_pattern = if is_wildcard {
240
+
nsid.strip_suffix(".*").unwrap()
241
+
} else {
242
+
nsid
243
+
};
244
+
245
+
// Check if already cached (for specific NSIDs only)
246
+
if !is_wildcard && cache.lexicons.contains_key(nsid) {
236
247
println!("Lexicon '{}' is already cached. Skipping fetch.", nsid);
237
248
println!(" (Use --force to re-fetch)");
238
249
return Ok(());
239
250
}
240
251
241
-
// Extract authority from NSID (e.g., "stream.place" from "stream.place.foo")
242
-
let authority = extract_authority(nsid)?;
243
-
println!("Fetching lexicons for authority: {}", authority);
252
+
// Extract authority from NSID (e.g., "place.stream" from "place.stream.key")
253
+
let authority = extract_authority(nsid_pattern)?;
254
+
println!("Fetching lexicons for pattern: {}", nsid);
244
255
245
256
// Step 1: DNS TXT lookup
246
257
let did = resolve_lexicon_did(&authority)?;
···
257
268
)));
258
269
}
259
270
271
+
let mut processed_count = 0;
272
+
260
273
// Step 3: Process each record
261
274
for record in records {
262
275
// Extract NSID from record URI or value
263
276
let record_nsid = extract_nsid_from_record(&record)?;
264
277
265
-
// Only process records that match or are under the requested NSID authority
266
-
if !record_nsid.starts_with(&authority) {
278
+
// Match against pattern
279
+
let matches = if is_wildcard {
280
+
// Wildcard: match all records starting with the pattern
281
+
record_nsid.starts_with(nsid_pattern) && record_nsid.len() > nsid_pattern.len()
282
+
} else {
283
+
// Specific: exact match only
284
+
record_nsid == nsid
285
+
};
286
+
287
+
if !matches {
267
288
continue;
268
289
}
269
290
270
291
println!(" Processing: {}", record_nsid);
292
+
processed_count += 1;
271
293
272
294
// Save JSON file with directory structure
273
295
// e.g., "place.stream.key" -> "place/stream/key.json"
···
316
338
// Save cache
317
339
cache.save(&cache_file)?;
318
340
319
-
println!("✓ Successfully fetched lexicons for {}", nsid);
341
+
if processed_count == 0 {
342
+
return Err(FetchError::HttpError(format!(
343
+
"No lexicons matched pattern: {}",
344
+
nsid
345
+
)));
346
+
}
347
+
348
+
println!("✓ Successfully fetched {} lexicon(s) for {}", processed_count, nsid);
349
+
Ok(())
350
+
}
351
+
352
+
fn validate_nsid_format(nsid: &str) -> Result<(), FetchError> {
353
+
// Remove wildcard suffix for validation
354
+
let nsid_base = nsid.strip_suffix(".*").unwrap_or(nsid);
355
+
356
+
let parts: Vec<&str> = nsid_base.split('.').collect();
357
+
358
+
// NSID must have at least 3 segments (authority + name)
359
+
// e.g., "place.stream.key" or "place.stream.*"
360
+
if parts.len() < 3 {
361
+
return Err(FetchError::InvalidNsid(format!(
362
+
"NSID must have at least 3 segments or use wildcard (e.g., 'place.stream.key' or 'place.stream.*'): {}",
363
+
nsid
364
+
)));
365
+
}
366
+
320
367
Ok(())
321
368
}
322
369
323
-
fn extract_authority(nsid: &str) -> Result<String, FetchError> {
370
+
fn extract_authority(nsid_pattern: &str) -> Result<String, FetchError> {
324
371
// NSID format: authority.name(.name)*
325
-
// For "stream.place" or "stream.place.foo", authority is "stream.place"
372
+
// For "place.stream.key", authority is "place.stream"
326
373
// Typically authority is the first 2 segments (reversed domain)
327
-
let parts: Vec<&str> = nsid.split('.').collect();
374
+
let parts: Vec<&str> = nsid_pattern.split('.').collect();
328
375
329
376
if parts.len() < 2 {
330
377
return Err(FetchError::InvalidNsid(format!(
331
378
"NSID must have at least 2 segments: {}",
332
-
nsid
379
+
nsid_pattern
333
380
)));
334
381
}
335
382
+101
-70
website/content/docs/cli/07-fetch.md
+101
-70
website/content/docs/cli/07-fetch.md
···
12
12
# Fetch all dependencies from mlf.toml
13
13
mlf fetch
14
14
15
-
# Fetch a specific namespace
16
-
mlf fetch <NAMESPACE>
15
+
# Fetch a specific lexicon
16
+
mlf fetch <NSID>
17
+
18
+
# Fetch all lexicons matching a wildcard
19
+
mlf fetch <PATTERN>
17
20
18
21
# Fetch and save to dependencies
19
-
mlf fetch <NAMESPACE> --save
22
+
mlf fetch <NSID> --save
20
23
```
21
24
22
25
**Arguments:**
23
-
- `[NAMESPACE]` - Optional namespace to fetch (e.g., `stream.place`, `app.bsky`)
26
+
- `[NSID]` - Optional NSID or pattern to fetch:
27
+
- Specific lexicon: `com.example.forum.post`
28
+
- Wildcard pattern: `com.example.forum.*`
29
+
- Real-world example: `app.bsky.feed.*`
24
30
25
31
**Options:**
26
-
- `--save` - Add the namespace to dependencies in `mlf.toml`
32
+
- `--save` - Add the NSID/pattern to dependencies in `mlf.toml`
27
33
28
34
## How It Works
29
35
···
42
48
43
49
```toml
44
50
[dependencies]
45
-
dependencies = ["stream.place", "app.bsky"]
51
+
dependencies = ["com.example.forum.*", "com.example.social.*"]
46
52
```
47
53
48
54
Run:
···
55
61
```
56
62
Fetching 2 dependencies...
57
63
58
-
Fetching: stream.place
59
-
Fetching lexicons for authority: stream.place
60
-
→ Resolved DID: did:web:stream.place
61
-
→ Using PDS: https://stream.place
62
-
→ Found 5 lexicon record(s)
63
-
Processing: stream.place.thread
64
-
→ Saved JSON to .mlf/lexicons/json/stream.place.thread.json
65
-
→ Converted to MLF at .mlf/lexicons/mlf/stream.place.thread.mlf
66
-
✓ Successfully fetched lexicons for stream.place
64
+
Fetching: com.example.forum.*
65
+
Fetching lexicons for pattern: com.example.forum.*
66
+
→ Resolved DID: did:web:example.com
67
+
→ Using PDS: https://example.com
68
+
→ Found 3 lexicon record(s)
69
+
Processing: com.example.forum.post
70
+
→ Saved JSON to .mlf/lexicons/json/com/example/forum/post.json
71
+
→ Converted to MLF at .mlf/lexicons/mlf/com/example/forum/post.mlf
72
+
Processing: com.example.forum.thread
73
+
→ Saved JSON to .mlf/lexicons/json/com/example/forum/thread.json
74
+
→ Converted to MLF at .mlf/lexicons/mlf/com/example/forum/thread.mlf
75
+
✓ Successfully fetched 2 lexicon(s) for com.example.forum.*
67
76
68
-
Fetching: app.bsky
77
+
Fetching: com.example.social.*
69
78
...
70
79
71
80
✓ Successfully fetched all 2 dependencies
72
81
```
73
82
74
-
### Fetch Specific Namespace
83
+
### Fetch Specific Lexicon
75
84
76
85
```bash
77
-
mlf fetch stream.place
86
+
mlf fetch com.example.forum.post
78
87
```
79
88
80
-
This downloads all lexicons under the `stream.place` authority.
89
+
This downloads only the `com.example.forum.post` lexicon.
90
+
91
+
### Fetch with Wildcard
92
+
93
+
```bash
94
+
mlf fetch com.example.forum.*
95
+
```
96
+
97
+
This downloads all lexicons under the `com.example.forum` namespace.
81
98
82
99
### Fetch and Save
83
100
84
101
```bash
85
-
mlf fetch stream.place --save
102
+
mlf fetch com.example.forum.* --save
86
103
```
87
104
88
105
This:
89
-
1. Downloads the lexicons
90
-
2. Adds `"stream.place"` to the dependencies array in `mlf.toml`
106
+
1. Downloads all `com.example.forum.*` lexicons
107
+
2. Adds `"com.example.forum.*"` to the dependencies array in `mlf.toml`
91
108
3. Creates `mlf.toml` if it doesn't exist
92
109
93
110
## Storage Structure
···
100
117
├── .lexicon-cache.toml # Cache metadata
101
118
└── lexicons/
102
119
├── json/ # Original JSON lexicons
103
-
│ ├── stream.place.thread.json
104
-
│ └── app.bsky.actor.profile.json
120
+
│ ├── com/
121
+
│ │ └── example/
122
+
│ │ ├── forum/
123
+
│ │ │ ├── post.json
124
+
│ │ │ └── thread.json
125
+
│ │ └── social/
126
+
│ │ └── profile.json
127
+
│ └── app/
128
+
│ └── bsky/
129
+
│ └── feed/
130
+
│ └── post.json
105
131
└── mlf/ # Converted MLF format
106
-
├── stream.place.thread.mlf
107
-
└── app.bsky.actor.profile.mlf
132
+
├── com/
133
+
│ └── example/
134
+
│ ├── forum/
135
+
│ │ ├── post.mlf
136
+
│ │ └── thread.mlf
137
+
│ └── social/
138
+
│ └── profile.mlf
139
+
└── app/
140
+
└── bsky/
141
+
└── feed/
142
+
└── post.mlf
108
143
```
109
144
110
145
### Cache File
···
112
147
The `.lexicon-cache.toml` tracks what's been fetched:
113
148
114
149
```toml
115
-
[[lexicons.stream.place.thread]]
116
-
nsid = "stream.place.thread"
150
+
[[lexicons."com.example.forum.post"]]
151
+
nsid = "com.example.forum.post"
117
152
fetched_at = "2024-01-15T10:30:00Z"
118
-
did = "did:web:stream.place"
153
+
did = "did:web:example.com"
154
+
hash = "abc123..."
119
155
```
120
156
121
157
## DNS Resolution
122
158
123
-
For a namespace like `stream.place.thread`:
159
+
For an NSID like `com.example.forum.post`:
124
160
125
-
1. Extract authority: `stream.place`
126
-
2. Reverse for DNS: `place.stream`
127
-
3. Query TXT record: `_lexicon.place.stream`
161
+
1. Extract authority: `com.example` (first 2 segments)
162
+
2. Reverse for DNS: `example.com`
163
+
3. Query TXT record: `_lexicon.example.com`
128
164
4. Parse `did=did:web:...` or `did=did:plc:...`
129
165
130
166
**Example DNS record:**
131
167
```
132
-
_lexicon.place.stream. 300 IN TXT "did=did:web:stream.place"
168
+
_lexicon.example.com. 300 IN TXT "did=did:web:example.com"
133
169
```
134
170
135
171
## DID Resolution
136
172
137
173
### did:web
138
174
139
-
For `did:web:stream.place`, the PDS is `https://stream.place`
175
+
For `did:web:example.com`, the PDS is `https://example.com`
140
176
141
177
### did:plc
142
178
···
149
185
### Type References
150
186
151
187
```mlf
152
-
use stream.place.thread;
188
+
use com.example.forum.post;
153
189
154
-
def Reply = {
155
-
thread!: stream.place.thread,
190
+
record reply {
191
+
post!: com.example.forum.post,
156
192
text!: string,
157
-
};
193
+
}
158
194
```
159
195
160
196
### Code Generation
···
178
214
If you don't have an `mlf.toml`, the fetch command will offer to create one:
179
215
180
216
```bash
181
-
$ mlf fetch stream.place
217
+
$ mlf fetch com.example.forum.*
182
218
No mlf.toml found in current or parent directories.
183
219
Would you like to create one in the current directory? (y/n)
184
220
y
···
188
224
189
225
## Re-fetching
190
226
191
-
If a namespace is already cached, fetch skips it:
227
+
If a lexicon is already cached, fetch skips it:
192
228
193
229
```bash
194
-
$ mlf fetch stream.place
195
-
Lexicon 'stream.place.thread' is already cached. Skipping fetch.
230
+
$ mlf fetch com.example.forum.post
231
+
Lexicon 'com.example.forum.post' is already cached. Skipping fetch.
196
232
(Use --force to re-fetch)
197
233
```
198
234
199
235
To re-fetch:
200
236
201
237
```bash
202
-
mlf fetch stream.place --force # Not yet implemented
238
+
mlf fetch com.example.forum.post --force # Not yet implemented
203
239
```
204
240
205
241
## Error Handling
···
207
243
### DNS Errors
208
244
209
245
```
210
-
✗ DNS lookup failed: No TXT record found for _lexicon.place.stream
246
+
✗ DNS lookup failed: No TXT record found for _lexicon.example.com
211
247
```
212
248
213
249
**Causes:**
···
229
265
### No Records Found
230
266
231
267
```
232
-
✗ No lexicon records found for stream.place
268
+
✗ No lexicons matched pattern: com.example.forum.*
233
269
```
234
270
235
271
**Causes:**
236
-
- Namespace exists but has no published lexicons
237
-
- Wrong namespace (typo)
272
+
- No lexicons exist matching the pattern
273
+
- Wrong NSID (typo)
238
274
- PDS doesn't support lexicon publishing
239
275
276
+
### Invalid NSID Format
277
+
278
+
```
279
+
✗ NSID must have at least 3 segments or use wildcard (e.g., 'com.example.forum.post' or 'com.example.forum.*'): com.example
280
+
```
281
+
282
+
**Solution:**
283
+
- Use a specific NSID: `com.example.forum.post`
284
+
- Or use a wildcard: `com.example.forum.*`
285
+
240
286
## Best Practices
241
287
242
288
1. **Fetch before work** - Always fetch dependencies before coding
···
245
291
4. **Check DNS** - Verify TXT records before fetching
246
292
5. **Version dependencies** - Consider tracking lexicon versions (future feature)
247
293
248
-
## CI/CD Integration
249
-
250
-
In your CI pipeline:
251
-
252
-
```yaml
253
-
# GitHub Actions example
254
-
- name: Fetch ATProto Lexicons
255
-
run: |
256
-
mlf fetch
257
-
258
-
- name: Generate Code
259
-
run: |
260
-
mlf generate
261
-
```
262
-
263
-
This ensures builds have access to the latest lexicons.
264
294
265
295
## Comparison with npm/cargo
266
296
···
279
309
280
310
```bash
281
311
# Check DNS resolution
282
-
dig TXT _lexicon.place.stream
312
+
dig TXT _lexicon.example.com
283
313
284
314
# Test DID resolution
285
315
curl https://plc.directory/did:plc:abc123
286
316
```
287
317
288
-
### Invalid Namespace
318
+
### Invalid NSID Format
289
319
290
-
Make sure you're using the correct namespace format:
291
-
- ✓ `stream.place`
292
-
- ✓ `app.bsky`
293
-
- ✗ `stream.place.thread` (too specific)
320
+
Make sure you're using the correct format:
321
+
- ✓ `com.example.forum.post` (specific lexicon)
322
+
- ✓ `com.example.forum.*` (wildcard)
323
+
- ✓ `app.bsky.feed.*` (real-world wildcard)
324
+
- ✗ `com.example` (must be specific or use wildcard)
294
325
295
326
### Permission Errors
296
327