this repo has no description
clean up
Signed-off-by: Marius Kimmina <mar.kimmina@gmail.com>
+147
-329
+85
-214
cmd/leaflet-hugo-sync/main.go
+85
-214
cmd/leaflet-hugo-sync/main.go
···
30
30
}
31
31
32
32
ctx := context.Background()
33
-
33
+
34
34
// 1. Resolve Handle to DID using public resolver (bsky.social)
35
35
baseClient := atproto.NewClient("https://bsky.social")
36
36
did, err := baseClient.ResolveHandle(ctx, cfg.Source.Handle)
···
49
49
// 3. Connect to User's PDS
50
50
pdsClient := atproto.NewClient(pdsEndpoint)
51
51
52
-
// 4. Resolve Publication (if configured)
53
-
54
-
var publicationURI string
55
-
56
-
if cfg.Source.PublicationName != "" {
57
-
58
-
fmt.Printf("Resolving publication '%s'...\n", cfg.Source.PublicationName)
59
-
60
-
pubRecords, err := pdsClient.FetchEntries(ctx, did, "pub.leaflet.publication")
61
-
62
-
if err != nil {
63
-
64
-
log.Fatalf("failed to fetch publications: %v", err)
65
-
66
-
}
67
-
68
-
for _, rec := range pubRecords {
52
+
// 4. Resolve Publication (if configured)
53
+
var publicationURI string
54
+
if cfg.Source.PublicationName != "" {
55
+
fmt.Printf("Resolving publication '%s'...\n", cfg.Source.PublicationName)
56
+
pubRecords, err := pdsClient.FetchEntries(ctx, did, "pub.leaflet.publication")
57
+
if err != nil {
58
+
log.Fatalf("failed to fetch publications: %v", err)
59
+
}
69
60
70
-
var pub atproto.LeafletPublication
71
-
72
-
if err := json.Unmarshal(rec.Value, &pub); err == nil {
73
-
74
-
if pub.Name == cfg.Source.PublicationName {
75
-
76
-
publicationURI = rec.Uri
77
-
78
-
fmt.Printf("Found publication URI: %s\n", publicationURI)
79
-
80
-
break
81
-
82
-
}
83
-
61
+
for _, rec := range pubRecords {
62
+
var pub atproto.LeafletPublication
63
+
if err := json.Unmarshal(rec.Value, &pub); err == nil {
64
+
if pub.Name == cfg.Source.PublicationName {
65
+
publicationURI = rec.Uri
66
+
fmt.Printf("Found publication URI: %s\n", publicationURI)
67
+
break
84
68
}
85
-
86
69
}
70
+
}
87
71
88
-
if publicationURI == "" {
89
-
90
-
log.Fatalf("Publication '%s' not found", cfg.Source.PublicationName)
91
-
92
-
}
93
-
72
+
if publicationURI == "" {
73
+
log.Fatalf("Publication '%s' not found", cfg.Source.PublicationName)
94
74
}
75
+
}
95
76
96
-
77
+
// 5. Fetch Entries
78
+
// Update collection to Leaflet Document if user hasn't specified it
79
+
collection := cfg.Source.Collection
80
+
if collection == "com.whtwnd.blog.entry" {
81
+
fmt.Println("Warning: Defaulting to 'pub.leaflet.document' as 'com.whtwnd.blog.entry' seems deprecated/unused for Leaflet.")
82
+
collection = "pub.leaflet.document"
83
+
}
97
84
98
-
// 5. Fetch Entries
85
+
records, err := pdsClient.FetchEntries(ctx, did, collection)
86
+
if err != nil {
87
+
log.Fatalf("failed to fetch entries: %v", err)
88
+
}
99
89
100
-
// Update collection to Leaflet Document if user hasn't specified it
90
+
fmt.Printf("Found %d entries\n", len(records))
101
91
102
-
collection := cfg.Source.Collection
92
+
downloader := media.NewDownloader(cfg.Output.ImagesDir, cfg.Output.ImagePathPrefix, pdsClient.XRPC.Host)
93
+
gen := generator.NewGenerator(cfg)
94
+
conv := converter.NewConverter()
103
95
104
-
if collection == "com.whtwnd.blog.entry" {
96
+
for _, rec := range records {
97
+
// Try to unmarshal as LeafletDocument
98
+
var doc atproto.LeafletDocument
105
99
106
-
fmt.Println("Warning: Defaulting to 'pub.leaflet.document' as 'com.whtwnd.blog.entry' seems deprecated/unused for Leaflet.")
100
+
// Check type first
101
+
var typeCheck struct {
102
+
Type string `json:"$type"`
103
+
}
104
+
if err := json.Unmarshal(rec.Value, &typeCheck); err != nil {
105
+
fmt.Printf("Failed to check type for record %s: %v\n", rec.Uri, err)
106
+
continue
107
+
}
107
108
108
-
collection = "pub.leaflet.document"
109
+
if typeCheck.Type != "pub.leaflet.document" {
110
+
// Skip or try legacy
111
+
continue
112
+
}
109
113
114
+
if err := json.Unmarshal(rec.Value, &doc); err != nil {
115
+
fmt.Printf("Failed to unmarshal record %s: %v\n", rec.Uri, err)
116
+
continue
110
117
}
111
118
112
-
119
+
// Filter by Publication
120
+
if publicationURI != "" && doc.Publication != publicationURI {
121
+
continue
122
+
}
113
123
114
-
records, err := pdsClient.FetchEntries(ctx, did, collection)
124
+
fmt.Printf("Processing: %s\n", doc.Title)
115
125
126
+
// Convert to Markdown
127
+
result, err := conv.ConvertLeaflet(&doc)
116
128
if err != nil {
117
-
118
-
log.Fatalf("failed to fetch entries: %v", err)
119
-
129
+
fmt.Printf(" Failed to convert document: %v\n", err)
130
+
continue
120
131
}
121
132
122
-
123
-
124
-
fmt.Printf("Found %d entries\n", len(records))
125
-
126
-
127
-
128
-
downloader := media.NewDownloader(cfg.Output.ImagesDir, cfg.Output.ImagePathPrefix, pdsClient.XRPC.Host)
129
-
130
-
gen := generator.NewGenerator(cfg)
131
-
132
-
conv := converter.NewConverter()
133
-
134
-
135
-
136
-
for _, rec := range records {
137
-
138
-
// Try to unmarshal as LeafletDocument
139
-
140
-
var doc atproto.LeafletDocument
141
-
142
-
143
-
144
-
// Check type first
145
-
146
-
var typeCheck struct {
147
-
148
-
Type string `json:"$type"`
149
-
150
-
}
151
-
152
-
if err := json.Unmarshal(rec.Value, &typeCheck); err != nil {
153
-
154
-
fmt.Printf("Failed to check type for record %s: %v\n", rec.Uri, err)
155
-
156
-
continue
157
-
158
-
}
159
-
160
-
161
-
162
-
if typeCheck.Type != "pub.leaflet.document" {
163
-
164
-
// Skip or try legacy
165
-
166
-
continue
167
-
168
-
}
169
-
170
-
171
-
172
-
if err := json.Unmarshal(rec.Value, &doc); err != nil {
173
-
174
-
fmt.Printf("Failed to unmarshal record %s: %v\n", rec.Uri, err)
175
-
176
-
continue
177
-
178
-
}
179
-
180
-
181
-
182
-
// Filter by Publication
183
-
184
-
if publicationURI != "" && doc.Publication != publicationURI {
185
-
186
-
// Skip entries not matching the publication
187
-
188
-
continue
189
-
190
-
}
191
-
192
-
193
-
194
-
fmt.Printf("Processing: %s\n", doc.Title)
195
-
196
-
197
-
198
-
// Convert to Markdown
199
-
200
-
result, err := conv.ConvertLeaflet(&doc)
201
-
133
+
// Download Images
134
+
finalContent := result.Markdown
135
+
for _, imgRef := range result.Images {
136
+
localPath, err := downloader.DownloadBlob(ctx, did, imgRef.Blob.Ref.Link)
202
137
if err != nil {
203
-
204
-
fmt.Printf(" Failed to convert document: %v\n", err)
205
-
138
+
fmt.Printf(" Failed to download image: %v\n", err)
206
139
continue
207
-
208
140
}
141
+
finalContent = strings.ReplaceAll(finalContent, imgRef.Blob.Ref.Link, localPath)
142
+
}
209
143
210
-
144
+
// Slug generation
145
+
slug := lastPathPart(rec.Uri)
211
146
212
-
// Download Images
147
+
// Construct original URL
148
+
originalURL := fmt.Sprintf("https://leaflet.pub/%s", slug)
213
149
214
-
finalContent := result.Markdown
215
-
216
-
for _, imgRef := range result.Images {
217
-
218
-
localPath, err := downloader.DownloadBlob(ctx, did, imgRef.Blob.Ref.Link)
150
+
postData := generator.PostData{
151
+
Title: doc.Title,
152
+
CreatedAt: doc.PublishedAt,
153
+
Slug: slug,
154
+
Handle: cfg.Source.Handle,
155
+
OriginalURL: originalURL,
156
+
Content: finalContent,
157
+
}
219
158
220
-
if err != nil {
221
-
222
-
fmt.Printf(" Failed to download image: %v\n", err)
223
-
224
-
continue
225
-
226
-
}
227
-
228
-
finalContent = strings.ReplaceAll(finalContent, imgRef.Blob.Ref.Link, localPath)
229
-
230
-
}
231
-
232
-
233
-
234
-
// Slug generation
235
-
236
-
slug := lastPathPart(rec.Uri)
237
-
238
-
239
-
240
-
// Construct Original URL (Assuming toolbox.leaflet.pub structure or generic)
241
-
242
-
// User config usually handles the "base" part in template.
243
-
244
-
// But here we construct a full URL for the OriginalURL field.
245
-
246
-
// If we don't know the base, we can't fully construct it.
247
-
248
-
// However, user provided "https://toolbox.leaflet.pub/{{ .Slug }}" in template.
249
-
250
-
// Let's pass the slug and handle.
251
-
252
-
// Ideally, we should construct the full URL if we can.
253
-
254
-
// For now, let's just pass what we have.
255
-
256
-
257
-
258
-
originalURL := fmt.Sprintf("https://leaflet.pub/%s", slug) // Fallback generic
259
-
260
-
261
-
262
-
postData := generator.PostData{
263
-
264
-
Title: doc.Title,
265
-
266
-
CreatedAt: doc.PublishedAt,
267
-
268
-
Slug: slug,
269
-
270
-
Handle: cfg.Source.Handle,
271
-
272
-
OriginalURL: originalURL,
273
-
274
-
Content: finalContent,
275
-
276
-
}
277
-
278
-
279
-
280
-
if err := gen.GeneratePost(postData); err != nil {
281
-
282
-
fmt.Printf(" Failed to generate post: %v\n", err)
283
-
284
-
}
285
-
159
+
if err := gen.GeneratePost(postData); err != nil {
160
+
fmt.Printf(" Failed to generate post: %v\n", err)
286
161
}
287
-
288
-
289
-
290
-
fmt.Println("Done!")
291
-
292
162
}
293
163
294
-
164
+
fmt.Println("Done!")
165
+
}
-28
internal/atproto/client.go
-28
internal/atproto/client.go
···
96
96
}
97
97
98
98
func (c *Client) FetchEntries(ctx context.Context, repo string, collection string) ([]Record, error) {
99
-
100
99
var records []Record
101
-
102
100
cursor := ""
103
-
104
-
105
101
106
102
for {
107
-
108
103
params := map[string]interface{}{
109
-
110
104
"repo": repo,
111
-
112
105
"collection": collection,
113
-
114
106
"limit": 100,
115
-
116
107
}
117
-
118
108
if cursor != "" {
119
-
120
109
params["cursor"] = cursor
121
-
122
110
}
123
-
124
-
125
111
126
112
var out ListRecordsResponse
127
-
128
113
if err := c.XRPC.Do(ctx, xrpc.Query, "", "com.atproto.repo.listRecords", params, nil, &out); err != nil {
129
-
130
114
return nil, fmt.Errorf("listing records: %w", err)
131
-
132
115
}
133
116
134
-
135
-
136
117
records = append(records, out.Records...)
137
-
138
-
139
118
140
119
if out.Cursor == nil || *out.Cursor == "" {
141
-
142
120
break
143
-
144
121
}
145
-
146
122
cursor = *out.Cursor
147
-
148
123
}
149
124
150
-
151
-
152
125
return records, nil
153
-
154
126
}
+12
-12
internal/atproto/types.go
+12
-12
internal/atproto/types.go
···
36
36
}
37
37
38
38
type Page struct {
39
-
Type string `json:"$type"` // pub.leaflet.pages.linearDocument
39
+
Type string `json:"$type"` // pub.leaflet.pages.linearDocument
40
40
Blocks []BlockWrapper `json:"blocks"`
41
41
}
42
42
···
68
68
}
69
69
70
70
type ListItem struct {
71
-
Type string `json:"$type"` // pub.leaflet.blocks.unorderedList#listItem
72
-
Content json.RawMessage `json:"content"` // Usually a TextBlock
73
-
Children []ListItem `json:"children"` // Nested lists?
71
+
Type string `json:"$type"` // pub.leaflet.blocks.unorderedList#listItem
72
+
Content json.RawMessage `json:"content"` // Usually a TextBlock
73
+
Children []ListItem `json:"children"` // Nested lists?
74
74
}
75
75
76
76
type ImageBlock struct {
···
80
80
}
81
81
82
82
type BskyPostBlock struct {
83
-
Type string `json:"$type"`
84
-
PostRef PostRef `json:"postRef"`
83
+
Type string `json:"$type"`
84
+
PostRef PostRef `json:"postRef"`
85
85
}
86
86
87
87
type PostRef struct {
88
-
Uri string `json:"uri"`
89
-
Cid string `json:"cid"`
88
+
Uri string `json:"uri"`
89
+
Cid string `json:"cid"`
90
90
}
91
91
92
92
// Shared Types
···
108
108
}
109
109
110
110
type Embed struct {
111
-
Type string `json:"$type"`
112
-
Images []ImageEmbed `json:"images,omitempty"`
113
-
External *ExternalEmbed `json:"external,omitempty"`
111
+
Type string `json:"$type"`
112
+
Images []ImageEmbed `json:"images,omitempty"`
113
+
External *ExternalEmbed `json:"external,omitempty"`
114
114
}
115
115
116
116
type ImageEmbed struct {
···
133
133
134
134
type BlobRef struct {
135
135
Link string `json:"$link"`
136
-
}
136
+
}
+50
-74
internal/converter/markdown.go
+50
-74
internal/converter/markdown.go
···
9
9
)
10
10
11
11
type Converter struct {
12
-
// We might need to track image downloads here or just return image references?
13
-
// For now, let's just return the Markdown text and a list of image blobs to download.
12
+
// No state needed; conversion is stateless
14
13
}
15
14
16
15
type ConversionResult struct {
···
72
71
if err := json.Unmarshal(blockWrapper.Block, &imgBlock); err != nil {
73
72
continue
74
73
}
75
-
// We use a placeholder path that main.go will resolve
76
-
// Actually, main.go needs to know about this.
77
-
// Let's assume standard markdown image syntax: 
78
-
// The downloader will replace it or we pre-calculate the path?
79
-
// Better: Return the blob CID as the URL, and main.go does a string replace or we handle it here if we pass config.
80
-
// For now, let's use the blob ref link as the URL.
74
+
// Use blob CID as placeholder URL; main.go replaces it with the local path
81
75
sb.WriteString(fmt.Sprintf("\n\n", imgBlock.Alt, imgBlock.Image.Ref.Link))
82
76
images = append(images, ImageRef{Blob: imgBlock.Image, Alt: imgBlock.Alt})
83
77
84
-
case "pub.leaflet.blocks.bskyPost":
85
-
var postBlock atproto.BskyPostBlock
86
-
if err := json.Unmarshal(blockWrapper.Block, &postBlock); err != nil {
87
-
continue
88
-
}
89
-
// Render a link to the post for now, as we can't easily embed it without JS
90
-
postURL := fmt.Sprintf("https://bsky.app/profile/%s/post/%s", "did:...", lastPathPart(postBlock.PostRef.Uri))
91
-
// We don't have the handle here easily to make a pretty URL, but we can try.
92
-
// Actually, let's just make a blockquote link.
93
-
sb.WriteString(fmt.Sprintf("> [View on Bluesky](%s)\n\n", postURL)) // TODO: Improve this
78
+
case "pub.leaflet.blocks.bskyPost":
79
+
var postBlock atproto.BskyPostBlock
80
+
if err := json.Unmarshal(blockWrapper.Block, &postBlock); err != nil {
81
+
continue
82
+
}
83
+
// Render as a blockquote link to the Bluesky post
84
+
postURL := fmt.Sprintf("https://bsky.app/profile/%s/post/%s", "did:...", lastPathPart(postBlock.PostRef.Uri))
85
+
sb.WriteString(fmt.Sprintf("> [View on Bluesky](%s)\n\n", postURL))
94
86
}
95
87
}
96
88
}
···
102
94
}
103
95
104
96
func (c *Converter) renderText(block *atproto.TextBlock) string {
105
-
// Apply facets
106
-
// Facets are ranges. We need to insert markdown syntax at specific indices.
107
-
// This is tricky because inserting characters shifts indices.
108
-
// Best approach: Slice the string and reconstruct it.
97
+
// Apply facets to convert rich text to markdown
98
+
// Note: ATProto facets use byte offsets, not rune offsets
99
+
data := []byte(block.Plaintext)
109
100
110
-
// Sort facets by start index (descending) to avoid shifting issues?
111
-
// Actually, we should iterate from start to end, keeping track of current index.
112
-
113
-
// Simplify: Just handle links for now.
114
-
// NOTE: Facets in ATProto are byte-offsets, not rune-offsets. Go strings are UTF-8.
115
-
116
-
// Convert string to byte slice for easier indexing
117
-
data := []byte(block.Plaintext)
118
-
119
-
// Map of byte_index -> string_to_insert
120
-
// But we wrap text.
121
-
122
-
// Let's just do a linear pass if facets are non-overlapping and sorted.
123
-
// They should be.
124
-
125
-
var sb strings.Builder
126
-
lastPos := 0
127
-
128
-
for _, facet := range block.Facets {
129
-
start := facet.Index.ByteStart
130
-
end := facet.Index.ByteEnd
131
-
132
-
if start < lastPos {
133
-
continue // Overlap or out of order
134
-
}
135
-
136
-
// Append text before facet
137
-
sb.Write(data[lastPos:start])
138
-
139
-
// Handle feature
140
-
text := string(data[start:end])
141
-
replacement := text
142
-
143
-
for _, feat := range facet.Features {
144
-
if feat.Type == "pub.leaflet.richtext.facet#link" {
145
-
replacement = fmt.Sprintf("[%s](%s)", text, feat.URI)
146
-
} else if feat.Type == "pub.leaflet.richtext.facet#didMention" {
147
-
replacement = fmt.Sprintf("[%s](https://bsky.app/profile/%s)", text, feat.Did)
148
-
}
149
-
// code facet? pub.leaflet.richtext.facet#code -> `text`
150
-
if feat.Type == "pub.leaflet.richtext.facet#code" {
151
-
replacement = fmt.Sprintf("`%s`", text)
152
-
}
153
-
}
154
-
155
-
sb.WriteString(replacement)
156
-
lastPos = end
157
-
}
158
-
159
-
sb.Write(data[lastPos:])
160
-
161
-
return sb.String()
101
+
var sb strings.Builder
102
+
lastPos := 0
103
+
104
+
for _, facet := range block.Facets {
105
+
start := facet.Index.ByteStart
106
+
end := facet.Index.ByteEnd
107
+
108
+
if start < lastPos {
109
+
continue // Overlap or out of order
110
+
}
111
+
112
+
// Append text before facet
113
+
sb.Write(data[lastPos:start])
114
+
115
+
// Handle feature
116
+
text := string(data[start:end])
117
+
replacement := text
118
+
119
+
for _, feat := range facet.Features {
120
+
if feat.Type == "pub.leaflet.richtext.facet#link" {
121
+
replacement = fmt.Sprintf("[%s](%s)", text, feat.URI)
122
+
} else if feat.Type == "pub.leaflet.richtext.facet#didMention" {
123
+
replacement = fmt.Sprintf("[%s](https://bsky.app/profile/%s)", text, feat.Did)
124
+
}
125
+
// code facet? pub.leaflet.richtext.facet#code -> `text`
126
+
if feat.Type == "pub.leaflet.richtext.facet#code" {
127
+
replacement = fmt.Sprintf("`%s`", text)
128
+
}
129
+
}
130
+
131
+
sb.WriteString(replacement)
132
+
lastPos = end
133
+
}
134
+
135
+
sb.Write(data[lastPos:])
136
+
137
+
return sb.String()
162
138
}
163
139
164
140
func (c *Converter) renderList(sb *strings.Builder, items []atproto.ListItem, depth int) {