A better Rust ATProto crate
schema discovery
+284
-17
+269
-10
crates/jacquard-lexgen/src/schema_discovery.rs
+269
-10
crates/jacquard-lexgen/src/schema_discovery.rs
···
9
9
//! use jacquard_lexgen::schema_discovery::WorkspaceDiscovery;
10
10
//!
11
11
//! fn main() -> miette::Result<()> {
12
-
//! // Discover all schemas in workspace
13
-
//! let schemas = WorkspaceDiscovery::new()
14
-
//! .scan()?;
15
-
//!
16
-
//! println!("Found {} schemas", schemas.len());
17
-
//!
18
-
//! for schema in schemas {
19
-
//! println!(" {}: {}", schema.nsid, schema.source_path.display());
20
-
//! }
12
+
//! // Discover and generate schemas
13
+
//! WorkspaceDiscovery::new()
14
+
//! .verbose(true)
15
+
//! .generate_and_write("lexicons")?;
21
16
//!
22
17
//! Ok(())
23
18
//! }
24
19
//! ```
25
20
21
+
use jacquard_lexicon::lexicon::LexiconDoc;
26
22
use miette::{IntoDiagnostic, Result};
23
+
use std::collections::BTreeMap;
27
24
use std::path::{Path, PathBuf};
28
-
use syn::{Attribute, Item};
25
+
use syn::{Attribute, DeriveInput, Item};
29
26
30
27
/// Discovered schema type
31
28
#[derive(Debug, Clone)]
···
59
56
pub key: Option<String>,
60
57
}
61
58
59
+
/// Generated schema with full LexiconDoc
60
+
#[derive(Debug, Clone)]
61
+
pub struct GeneratedSchema {
62
+
/// The NSID from the generated schema
63
+
pub nsid: String,
64
+
/// The schema_id (may include fragment)
65
+
pub schema_id: String,
66
+
/// The generated lexicon document
67
+
pub doc: LexiconDoc<'static>,
68
+
/// Source file containing this type
69
+
pub source_path: PathBuf,
70
+
}
71
+
62
72
/// Workspace schema discovery via source scanning
63
73
pub struct WorkspaceDiscovery {
64
74
workspace_root: PathBuf,
···
118
128
Ok(schemas)
119
129
}
120
130
131
+
/// Scan workspace and generate complete schemas
132
+
pub fn scan_and_generate(&self) -> Result<Vec<GeneratedSchema>> {
133
+
let discovered = self.scan()?;
134
+
135
+
if self.verbose {
136
+
println!("Generating schemas for {} types...", discovered.len());
137
+
}
138
+
139
+
let mut generated = Vec::new();
140
+
141
+
for schema_info in discovered {
142
+
if self.verbose {
143
+
println!(
144
+
"Generating schema for {}: {}",
145
+
schema_info.type_name, schema_info.nsid
146
+
);
147
+
}
148
+
149
+
// Re-parse the source file to get full AST
150
+
let contents = std::fs::read_to_string(&schema_info.source_path).into_diagnostic()?;
151
+
let file = syn::parse_file(&contents).into_diagnostic()?;
152
+
153
+
// Find the specific type
154
+
let ast = self.find_type_in_file(&file, &schema_info.type_name)?;
155
+
156
+
// Use schema builder based on kind
157
+
let built = match schema_info.kind {
158
+
SchemaKind::Struct => {
159
+
jacquard_lexicon::schema::from_ast::build_struct_schema(&ast)?
160
+
}
161
+
SchemaKind::Enum => {
162
+
jacquard_lexicon::schema::from_ast::build_enum_schema(&ast)?
163
+
}
164
+
};
165
+
166
+
generated.push(GeneratedSchema {
167
+
nsid: built.nsid,
168
+
schema_id: built.schema_id,
169
+
doc: built.doc,
170
+
source_path: schema_info.source_path.clone(),
171
+
});
172
+
}
173
+
174
+
if self.verbose {
175
+
println!("Generated {} schemas", generated.len());
176
+
}
177
+
178
+
Ok(generated)
179
+
}
180
+
181
+
/// Generate schemas and write to directory
182
+
pub fn generate_and_write(&self, output_dir: impl AsRef<Path>) -> Result<()> {
183
+
let schemas = self.scan_and_generate()?;
184
+
185
+
if schemas.is_empty() {
186
+
println!("No schemas found to generate");
187
+
return Ok(());
188
+
}
189
+
190
+
// Group by base NSID (strip #fragments)
191
+
let grouped = self.group_by_base_nsid(&schemas);
192
+
193
+
// Create output directory
194
+
std::fs::create_dir_all(output_dir.as_ref()).into_diagnostic()?;
195
+
196
+
// Write each group
197
+
let mut written = 0;
198
+
for (base_nsid, group) in &grouped {
199
+
self.write_lexicon_file(output_dir.as_ref(), base_nsid, group)?;
200
+
written += 1;
201
+
}
202
+
203
+
println!(
204
+
"✓ Wrote {} lexicon files to {}",
205
+
written,
206
+
output_dir.as_ref().display()
207
+
);
208
+
209
+
Ok(())
210
+
}
211
+
212
+
/// Group schemas by base NSID (strip fragment suffix)
213
+
fn group_by_base_nsid(&self, schemas: &[GeneratedSchema]) -> BTreeMap<String, Vec<&GeneratedSchema>> {
214
+
let mut groups: BTreeMap<String, Vec<&GeneratedSchema>> = BTreeMap::new();
215
+
216
+
for schema in schemas {
217
+
// Split on # to get base NSID
218
+
let base_nsid = if let Some(pos) = schema.nsid.find('#') {
219
+
&schema.nsid[..pos]
220
+
} else {
221
+
&schema.nsid
222
+
};
223
+
224
+
groups
225
+
.entry(base_nsid.to_string())
226
+
.or_default()
227
+
.push(schema);
228
+
}
229
+
230
+
groups
231
+
}
232
+
233
+
/// Write a single lexicon file
234
+
fn write_lexicon_file(
235
+
&self,
236
+
output_dir: &Path,
237
+
base_nsid: &str,
238
+
schemas: &[&GeneratedSchema],
239
+
) -> Result<()> {
240
+
use jacquard_lexicon::lexicon::Lexicon;
241
+
242
+
// Merge all defs into one LexiconDoc
243
+
let mut all_defs = BTreeMap::new();
244
+
let mut primary_doc: Option<LexiconDoc> = None;
245
+
246
+
for schema in schemas {
247
+
// Determine if this is the primary def or a fragment
248
+
if schema.nsid.contains('#') {
249
+
// Fragment - extract def name and add to defs
250
+
let fragment_name = schema.nsid.split('#').nth(1).unwrap();
251
+
252
+
// Merge defs from fragment doc
253
+
for (def_name, def) in &schema.doc.defs {
254
+
// Use fragment name if def is "main", otherwise use as-is
255
+
let final_name = if def_name == "main" {
256
+
fragment_name.to_string()
257
+
} else {
258
+
def_name.to_string()
259
+
};
260
+
all_defs.insert(final_name, def.clone());
261
+
}
262
+
} else {
263
+
// Primary type - use as base doc
264
+
primary_doc = Some(schema.doc.clone());
265
+
}
266
+
}
267
+
268
+
// Build final doc
269
+
let mut final_doc = primary_doc.unwrap_or_else(|| LexiconDoc {
270
+
lexicon: Lexicon::Lexicon1,
271
+
id: base_nsid.into(),
272
+
revision: None,
273
+
description: None,
274
+
defs: BTreeMap::new(),
275
+
});
276
+
277
+
// Merge in all defs
278
+
for (k, v) in all_defs {
279
+
final_doc.defs.insert(k.into(), v);
280
+
}
281
+
282
+
// Serialize to JSON with "main" def first
283
+
let json = self.serialize_with_main_first(&final_doc)?;
284
+
285
+
// Write to file
286
+
let filename = base_nsid.replace('.', "_") + ".json";
287
+
let path = output_dir.join(&filename);
288
+
289
+
std::fs::write(&path, json).into_diagnostic()?;
290
+
291
+
if self.verbose {
292
+
println!(" Wrote {} ({} defs)", filename, final_doc.defs.len());
293
+
}
294
+
295
+
Ok(())
296
+
}
297
+
298
+
/// Serialize a lexicon doc with "main" def first
299
+
fn serialize_with_main_first(&self, doc: &LexiconDoc) -> Result<String> {
300
+
use serde_json::{json, Map, Value};
301
+
302
+
// Build defs map with main first
303
+
let mut defs_map = Map::new();
304
+
305
+
// Insert main first if it exists
306
+
if let Some(main_def) = doc.defs.get("main") {
307
+
let main_value = serde_json::to_value(main_def).into_diagnostic()?;
308
+
defs_map.insert("main".to_string(), main_value);
309
+
}
310
+
311
+
// Insert all other defs in sorted order
312
+
for (name, def) in &doc.defs {
313
+
if name != "main" {
314
+
let def_value = serde_json::to_value(def).into_diagnostic()?;
315
+
defs_map.insert(name.to_string(), def_value);
316
+
}
317
+
}
318
+
319
+
// Build final JSON object
320
+
let mut obj = Map::new();
321
+
obj.insert("lexicon".to_string(), json!(1));
322
+
obj.insert("id".to_string(), json!(doc.id.as_ref()));
323
+
324
+
if let Some(rev) = &doc.revision {
325
+
obj.insert("revision".to_string(), json!(rev));
326
+
}
327
+
328
+
if let Some(desc) = &doc.description {
329
+
obj.insert("description".to_string(), json!(desc));
330
+
}
331
+
332
+
obj.insert("defs".to_string(), Value::Object(defs_map));
333
+
334
+
// Pretty-print JSON
335
+
serde_json::to_string_pretty(&Value::Object(obj)).into_diagnostic()
336
+
}
337
+
121
338
/// Find workspace members by parsing Cargo.toml
122
339
fn find_workspace_members(&self) -> Result<Vec<PathBuf>> {
123
340
let cargo_toml = self.workspace_root.join("Cargo.toml");
···
218
435
}
219
436
220
437
Ok(schemas)
438
+
}
439
+
440
+
/// Find a type in a parsed file and convert to DeriveInput
441
+
fn find_type_in_file(&self, file: &syn::File, type_name: &str) -> Result<DeriveInput> {
442
+
for item in &file.items {
443
+
match item {
444
+
Item::Struct(item_struct) if item_struct.ident == type_name => {
445
+
// Convert ItemStruct to DeriveInput
446
+
return Ok(DeriveInput {
447
+
attrs: item_struct.attrs.clone(),
448
+
vis: item_struct.vis.clone(),
449
+
ident: item_struct.ident.clone(),
450
+
generics: item_struct.generics.clone(),
451
+
data: syn::Data::Struct(syn::DataStruct {
452
+
struct_token: item_struct.struct_token,
453
+
fields: item_struct.fields.clone(),
454
+
semi_token: item_struct.semi_token,
455
+
}),
456
+
});
457
+
}
458
+
Item::Enum(item_enum) if item_enum.ident == type_name => {
459
+
// Convert ItemEnum to DeriveInput
460
+
return Ok(DeriveInput {
461
+
attrs: item_enum.attrs.clone(),
462
+
vis: item_enum.vis.clone(),
463
+
ident: item_enum.ident.clone(),
464
+
generics: item_enum.generics.clone(),
465
+
data: syn::Data::Enum(syn::DataEnum {
466
+
enum_token: item_enum.enum_token,
467
+
brace_token: item_enum.brace_token,
468
+
variants: item_enum.variants.clone(),
469
+
}),
470
+
});
471
+
}
472
+
_ => continue,
473
+
}
474
+
}
475
+
476
+
Err(miette::miette!(
477
+
"Type {} not found in source file",
478
+
type_name
479
+
))
221
480
}
222
481
223
482
/// Extract schema info from attributes
+15
-7
crates/jacquard-lexicon/src/derive_impl/doc_to_tokens.rs
+15
-7
crates/jacquard-lexicon/src/derive_impl/doc_to_tokens.rs
···
182
182
/// Convert LexObjectProperty to tokens
183
183
fn object_property_to_tokens(prop: &LexObjectProperty) -> TokenStream {
184
184
match prop {
185
-
LexObjectProperty::Boolean(b) => quote! {
185
+
LexObjectProperty::Boolean(_) => quote! {
186
186
::jacquard_lexicon::lexicon::LexObjectProperty::Boolean(
187
187
::jacquard_lexicon::lexicon::LexBoolean {
188
188
description: None,
···
374
374
LexStringFormat::AtUri => quote! { ::jacquard_lexicon::lexicon::LexStringFormat::AtUri },
375
375
LexStringFormat::Nsid => quote! { ::jacquard_lexicon::lexicon::LexStringFormat::Nsid },
376
376
LexStringFormat::Cid => quote! { ::jacquard_lexicon::lexicon::LexStringFormat::Cid },
377
-
LexStringFormat::Datetime => quote! { ::jacquard_lexicon::lexicon::LexStringFormat::Datetime },
378
-
LexStringFormat::Language => quote! { ::jacquard_lexicon::lexicon::LexStringFormat::Language },
377
+
LexStringFormat::Datetime => {
378
+
quote! { ::jacquard_lexicon::lexicon::LexStringFormat::Datetime }
379
+
}
380
+
LexStringFormat::Language => {
381
+
quote! { ::jacquard_lexicon::lexicon::LexStringFormat::Language }
382
+
}
379
383
LexStringFormat::Tid => quote! { ::jacquard_lexicon::lexicon::LexStringFormat::Tid },
380
-
LexStringFormat::RecordKey => quote! { ::jacquard_lexicon::lexicon::LexStringFormat::RecordKey },
381
-
LexStringFormat::AtIdentifier => quote! { ::jacquard_lexicon::lexicon::LexStringFormat::AtIdentifier },
384
+
LexStringFormat::RecordKey => {
385
+
quote! { ::jacquard_lexicon::lexicon::LexStringFormat::RecordKey }
386
+
}
387
+
LexStringFormat::AtIdentifier => {
388
+
quote! { ::jacquard_lexicon::lexicon::LexStringFormat::AtIdentifier }
389
+
}
382
390
LexStringFormat::Uri => quote! { ::jacquard_lexicon::lexicon::LexStringFormat::Uri },
383
391
});
384
392
let min_len = option_to_tokens(&s.min_length, |v| quote! { #v });
···
457
465
quote! {
458
466
::jacquard_lexicon::lexicon::LexXrpcParametersProperty::String(#string_tokens)
459
467
}
460
-
},
468
+
}
461
469
LexXrpcParametersProperty::Unknown(_) => quote! {
462
470
::jacquard_lexicon::lexicon::LexXrpcParametersProperty::Unknown(
463
471
::jacquard_lexicon::lexicon::LexUnknown { description: None }
···
510
518
}
511
519
)
512
520
}
513
-
},
521
+
}
514
522
}
515
523
}
516
524