+3

backend/cmd/server/main.go

reviewed

··· 41 41 } 42 42 43 43 ingester := firehose.NewIngester(database) 44 44 + firehose.RelayURL = getEnv("BLOCK_RELAY_URL", "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos") 45 45 + log.Printf("Firehose URL: %s", firehose.RelayURL) 46 46 + 44 47 go func() { 45 48 if err := ingester.Start(context.Background()); err != nil { 46 49 log.Printf("Firehose ingester error: %v", err)

+83 -25

backend/internal/api/annotations.go

reviewed

··· 22 22 } 23 23 24 24 type CreateAnnotationRequest struct { 25 25 - URL string `json:"url"` 26 26 - Text string `json:"text"` 27 27 - Selector interface{} `json:"selector,omitempty"` 28 28 - Title string `json:"title,omitempty"` 29 29 - Tags []string `json:"tags,omitempty"` 25 25 + URL string `json:"url"` 26 26 + Text string `json:"text"` 27 27 + Selector json.RawMessage `json:"selector,omitempty"` 28 28 + Title string `json:"title,omitempty"` 29 29 + Tags []string `json:"tags,omitempty"` 30 30 } 31 31 32 32 type CreateAnnotationResponse struct { ··· 77 77 } 78 78 79 79 var result *xrpc.CreateRecordOutput 80 80 + 81 81 + if existing, err := s.checkDuplicateAnnotation(session.DID, req.URL, req.Text); err == nil && existing != nil { 82 82 + w.Header().Set("Content-Type", "application/json") 83 83 + json.NewEncoder(w).Encode(CreateAnnotationResponse{ 84 84 + URI: existing.URI, 85 85 + CID: *existing.CID, 86 86 + }) 87 87 + return 88 88 + } 89 89 + 80 90 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 81 91 var createErr error 82 92 result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionAnnotation, record) ··· 237 247 return fmt.Errorf("failed to fetch existing record: %w", getErr) 238 248 } 239 249 240 240 - var record map[string]interface{} 250 250 + var record xrpc.AnnotationRecord 241 251 if err := json.Unmarshal(existing.Value, &record); err != nil { 242 252 return fmt.Errorf("failed to parse existing record: %w", err) 243 253 } 244 254 245 245 - record["text"] = req.Text 246 246 - if req.Tags != nil { 247 247 - record["tags"] = req.Tags 255 255 + record.Body = &xrpc.AnnotationBody{ 256 256 + Value: req.Text, 257 257 + Format: "text/plain", 258 258 + } 259 259 + if len(req.Tags) > 0 { 260 260 + record.Tags = req.Tags 248 261 } else { 249 249 - delete(record, "tags") 262 262 + record.Tags = nil 263 263 + } 264 264 + 265 265 + if err := record.Validate(); err != nil { 266 266 + return fmt.Errorf("validation failed: %w", err) 250 267 } 251 268 252 269 var updateErr error ··· 309 326 310 327 record := xrpc.NewLikeRecord(req.SubjectURI, req.SubjectCID) 311 328 329 329 + if err := record.Validate(); err != nil { 330 330 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 331 331 + return 332 332 + } 333 333 + 312 334 var result *xrpc.CreateRecordOutput 313 335 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 314 336 var createErr error ··· 402 424 } 403 425 404 426 record := xrpc.NewReplyRecord(req.ParentURI, req.ParentCID, req.RootURI, req.RootCID, req.Text) 427 427 + 428 428 + if err := record.Validate(); err != nil { 429 429 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 430 430 + return 431 431 + } 405 432 406 433 var result *xrpc.CreateRecordOutput 407 434 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { ··· 509 536 } 510 537 511 538 type CreateHighlightRequest struct { 512 512 - URL string `json:"url"` 513 513 - Title string `json:"title,omitempty"` 514 514 - Selector interface{} `json:"selector"` 515 515 - Color string `json:"color,omitempty"` 516 516 - Tags []string `json:"tags,omitempty"` 539 539 + URL string `json:"url"` 540 540 + Title string `json:"title,omitempty"` 541 541 + Selector json.RawMessage `json:"selector"` 542 542 + Color string `json:"color,omitempty"` 543 543 + Tags []string `json:"tags,omitempty"` 517 544 } 518 545 519 546 func (s *AnnotationService) CreateHighlight(w http.ResponseWriter, r *http.Request) { ··· 537 564 urlHash := db.HashURL(req.URL) 538 565 record := xrpc.NewHighlightRecord(req.URL, urlHash, req.Selector, req.Color, req.Tags) 539 566 567 567 + if err := record.Validate(); err != nil { 568 568 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 569 569 + return 570 570 + } 571 571 + 540 572 var result *xrpc.CreateRecordOutput 573 573 + 574 574 + if existing, err := s.checkDuplicateHighlight(session.DID, req.URL, req.Selector); err == nil && existing != nil { 575 575 + w.Header().Set("Content-Type", "application/json") 576 576 + json.NewEncoder(w).Encode(map[string]string{"uri": existing.URI, "cid": *existing.CID}) 577 577 + return 578 578 + } 579 579 + 541 580 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 542 581 var createErr error 543 582 result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionHighlight, record) ··· 549 588 } 550 589 551 590 var selectorJSONPtr *string 552 552 - if req.Selector != nil { 553 553 - selectorBytes, _ := json.Marshal(req.Selector) 554 554 - selectorStr := string(selectorBytes) 591 591 + if len(record.Target.Selector) > 0 { 592 592 + selectorStr := string(record.Target.Selector) 555 593 selectorJSONPtr = &selectorStr 556 594 } 557 595 ··· 622 660 urlHash := db.HashURL(req.URL) 623 661 record := xrpc.NewBookmarkRecord(req.URL, urlHash, req.Title, req.Description) 624 662 663 663 + if err := record.Validate(); err != nil { 664 664 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 665 665 + return 666 666 + } 667 667 + 625 668 var result *xrpc.CreateRecordOutput 669 669 + 670 670 + if existing, err := s.checkDuplicateBookmark(session.DID, req.URL); err == nil && existing != nil { 671 671 + w.Header().Set("Content-Type", "application/json") 672 672 + json.NewEncoder(w).Encode(map[string]string{"uri": existing.URI, "cid": *existing.CID}) 673 673 + return 674 674 + } 675 675 + 626 676 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 627 677 var createErr error 628 678 result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionBookmark, record) ··· 759 809 return fmt.Errorf("failed to fetch record: %w", getErr) 760 810 } 761 811 762 762 - var record map[string]interface{} 812 812 + var record xrpc.HighlightRecord 763 813 json.Unmarshal(existing.Value, &record) 764 814 765 815 if req.Color != "" { 766 766 - record["color"] = req.Color 816 816 + record.Color = req.Color 767 817 } 768 818 if req.Tags != nil { 769 769 - record["tags"] = req.Tags 819 819 + record.Tags = req.Tags 820 820 + } 821 821 + 822 822 + if err := record.Validate(); err != nil { 823 823 + return fmt.Errorf("validation failed: %w", err) 770 824 } 771 825 772 826 var updateErr error ··· 839 893 return fmt.Errorf("failed to fetch record: %w", getErr) 840 894 } 841 895 842 842 - var record map[string]interface{} 896 896 + var record xrpc.BookmarkRecord 843 897 json.Unmarshal(existing.Value, &record) 844 898 845 899 if req.Title != "" { 846 846 - record["title"] = req.Title 900 900 + record.Title = req.Title 847 901 } 848 902 if req.Description != "" { 849 849 - record["description"] = req.Description 903 903 + record.Description = req.Description 850 904 } 851 905 if req.Tags != nil { 852 852 - record["tags"] = req.Tags 906 906 + record.Tags = req.Tags 907 907 + } 908 908 + 909 909 + if err := record.Validate(); err != nil { 910 910 + return fmt.Errorf("validation failed: %w", err) 853 911 } 854 912 855 913 var updateErr error

+59

backend/internal/api/annotations_helpers.go

reviewed

··· 1 1 + package api 2 2 + 3 3 + import ( 4 4 + "encoding/json" 5 5 + "time" 6 6 + 7 7 + "margin.at/internal/db" 8 8 + ) 9 9 + 10 10 + func (s *AnnotationService) checkDuplicateAnnotation(did, url, text string) (*db.Annotation, error) { 11 11 + recentAnnos, err := s.db.GetAnnotationsByAuthor(did, 5, 0) 12 12 + if err != nil { 13 13 + return nil, err 14 14 + } 15 15 + for _, a := range recentAnnos { 16 16 + if a.TargetSource == url && 17 17 + ((a.BodyValue == nil && text == "") || (a.BodyValue != nil && *a.BodyValue == text)) && 18 18 + time.Since(a.CreatedAt) < 10*time.Second { 19 19 + return &a, nil 20 20 + } 21 21 + } 22 22 + return nil, nil 23 23 + } 24 24 + 25 25 + func (s *AnnotationService) checkDuplicateHighlight(did, url string, selector json.RawMessage) (*db.Highlight, error) { 26 26 + recentHighs, err := s.db.GetHighlightsByAuthor(did, 5, 0) 27 27 + if err != nil { 28 28 + return nil, err 29 29 + } 30 30 + for _, h := range recentHighs { 31 31 + matchSelector := false 32 32 + if h.SelectorJSON == nil && selector == nil { 33 33 + matchSelector = true 34 34 + } else if h.SelectorJSON != nil && selector != nil { 35 35 + selectorBytes, _ := json.Marshal(selector) 36 36 + if *h.SelectorJSON == string(selectorBytes) { 37 37 + matchSelector = true 38 38 + } 39 39 + } 40 40 + 41 41 + if h.TargetSource == url && matchSelector && time.Since(h.CreatedAt) < 10*time.Second { 42 42 + return &h, nil 43 43 + } 44 44 + } 45 45 + return nil, nil 46 46 + } 47 47 + 48 48 + func (s *AnnotationService) checkDuplicateBookmark(did, url string) (*db.Bookmark, error) { 49 49 + recentBooks, err := s.db.GetBookmarksByAuthor(did, 5, 0) 50 50 + if err != nil { 51 51 + return nil, err 52 52 + } 53 53 + for _, b := range recentBooks { 54 54 + if b.Source == url && time.Since(b.CreatedAt) < 10*time.Second { 55 55 + return &b, nil 56 56 + } 57 57 + } 58 58 + return nil, nil 59 59 + }

+114 -31

backend/internal/api/apikey.go

reviewed

··· 157 157 urlHash := db.HashURL(req.URL) 158 158 record := xrpc.NewBookmarkRecord(req.URL, urlHash, req.Title, req.Description) 159 159 160 160 + if err := record.Validate(); err != nil { 161 161 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 162 162 + return 163 163 + } 164 164 + 160 165 var result *xrpc.CreateRecordOutput 161 166 err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 162 167 var createErr error ··· 200 205 }) 201 206 } 202 207 203 203 - type QuickAnnotationRequest struct { 204 204 - URL string `json:"url"` 205 205 - Text string `json:"text"` 208 208 + type QuickSaveRequest struct { 209 209 + URL string `json:"url"` 210 210 + Text string `json:"text,omitempty"` 211 211 + Selector json.RawMessage `json:"selector,omitempty"` 212 212 + Color string `json:"color,omitempty"` 206 213 } 207 214 208 208 - func (h *APIKeyHandler) QuickAnnotation(w http.ResponseWriter, r *http.Request) { 215 215 + func (h *APIKeyHandler) QuickSave(w http.ResponseWriter, r *http.Request) { 209 216 apiKey, err := h.authenticateAPIKey(r) 210 217 if err != nil { 211 218 http.Error(w, err.Error(), http.StatusUnauthorized) 212 219 return 213 220 } 214 221 215 215 - var req QuickAnnotationRequest 222 222 + var req QuickSaveRequest 216 223 if err := json.NewDecoder(r.Body).Decode(&req); err != nil { 217 224 http.Error(w, "Invalid request body", http.StatusBadRequest) 218 225 return 219 226 } 220 227 221 221 - if req.URL == "" || req.Text == "" { 222 222 - http.Error(w, "URL and text are required", http.StatusBadRequest) 228 228 + if req.URL == "" { 229 229 + http.Error(w, "URL is required", http.StatusBadRequest) 223 230 return 224 231 } 225 232 ··· 230 237 } 231 238 232 239 urlHash := db.HashURL(req.URL) 233 233 - record := xrpc.NewAnnotationRecord(req.URL, urlHash, req.Text, nil, "") 240 240 + 241 241 + var isHighlight bool 242 242 + if req.Selector != nil && req.Text == "" { 243 243 + isHighlight = true 244 244 + } 234 245 235 246 var result *xrpc.CreateRecordOutput 236 236 - err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 237 237 - var createErr error 238 238 - result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionAnnotation, record) 239 239 - return createErr 240 240 - }) 241 241 - if err != nil { 242 242 - http.Error(w, "Failed to create annotation: "+err.Error(), http.StatusInternalServerError) 243 243 - return 244 244 - } 247 247 + var createErr error 248 248 + 249 249 + if isHighlight { 250 250 + color := req.Color 251 251 + if color == "" { 252 252 + color = "yellow" 253 253 + } 254 254 + record := xrpc.NewHighlightRecord(req.URL, urlHash, req.Selector, color, nil) 255 255 + 256 256 + if err := record.Validate(); err != nil { 257 257 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 258 258 + return 259 259 + } 260 260 + 261 261 + err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 262 262 + result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionHighlight, record) 263 263 + return createErr 264 264 + }) 265 265 + if err == nil { 266 266 + h.db.UpdateAPIKeyLastUsed(apiKey.ID) 267 267 + selectorJSON, _ := json.Marshal(req.Selector) 268 268 + selectorStr := string(selectorJSON) 269 269 + colorPtr := &color 270 270 + 271 271 + highlight := &db.Highlight{ 272 272 + URI: result.URI, 273 273 + AuthorDID: apiKey.OwnerDID, 274 274 + TargetSource: req.URL, 275 275 + TargetHash: urlHash, 276 276 + SelectorJSON: &selectorStr, 277 277 + Color: colorPtr, 278 278 + CreatedAt: time.Now(), 279 279 + IndexedAt: time.Now(), 280 280 + CID: &result.CID, 281 281 + } 282 282 + go func() { 283 283 + if err := h.db.CreateHighlight(highlight); err != nil { 284 284 + fmt.Printf("Warning: failed to index highlight in local DB: %v\n", err) 285 285 + } 286 286 + }() 287 287 + } 288 288 + 289 289 + } else { 290 290 + record := xrpc.NewAnnotationRecord(req.URL, urlHash, req.Text, req.Selector, "") 291 291 + 292 292 + if err := record.Validate(); err != nil { 293 293 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 294 294 + return 295 295 + } 245 296 246 246 - h.db.UpdateAPIKeyLastUsed(apiKey.ID) 297 297 + err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 298 298 + result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionAnnotation, record) 299 299 + return createErr 300 300 + }) 301 301 + if err == nil { 302 302 + h.db.UpdateAPIKeyLastUsed(apiKey.ID) 247 303 248 248 - bodyValue := req.Text 249 249 - annotation := &db.Annotation{ 250 250 - URI: result.URI, 251 251 - AuthorDID: apiKey.OwnerDID, 252 252 - Motivation: "commenting", 253 253 - BodyValue: &bodyValue, 254 254 - TargetSource: req.URL, 255 255 - TargetHash: urlHash, 256 256 - CreatedAt: time.Now(), 257 257 - IndexedAt: time.Now(), 258 258 - CID: &result.CID, 304 304 + var selectorStrPtr *string 305 305 + if req.Selector != nil { 306 306 + b, _ := json.Marshal(req.Selector) 307 307 + s := string(b) 308 308 + selectorStrPtr = &s 309 309 + } 310 310 + 311 311 + bodyValue := req.Text 312 312 + var bodyValuePtr *string 313 313 + if bodyValue != "" { 314 314 + bodyValuePtr = &bodyValue 315 315 + } 316 316 + 317 317 + annotation := &db.Annotation{ 318 318 + URI: result.URI, 319 319 + AuthorDID: apiKey.OwnerDID, 320 320 + Motivation: "commenting", 321 321 + BodyValue: bodyValuePtr, 322 322 + TargetSource: req.URL, 323 323 + TargetHash: urlHash, 324 324 + SelectorJSON: selectorStrPtr, 325 325 + CreatedAt: time.Now(), 326 326 + IndexedAt: time.Now(), 327 327 + CID: &result.CID, 328 328 + } 329 329 + go func() { 330 330 + h.db.CreateAnnotation(annotation) 331 331 + }() 332 332 + } 259 333 } 260 260 - h.db.CreateAnnotation(annotation) 334 334 + 335 335 + if err != nil { 336 336 + http.Error(w, "Failed to create record: "+err.Error(), http.StatusInternalServerError) 337 337 + return 338 338 + } 261 339 262 340 w.Header().Set("Content-Type", "application/json") 263 341 json.NewEncoder(w).Encode(map[string]string{ 264 342 "uri": result.URI, 265 343 "cid": result.CID, 266 266 - "message": "Annotation created successfully", 344 344 + "message": "Saved successfully", 267 345 }) 268 346 } 269 347 ··· 304 382 } 305 383 306 384 record := xrpc.NewHighlightRecord(req.URL, urlHash, req.Selector, color, nil) 385 385 + 386 386 + if err := record.Validate(); err != nil { 387 387 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 388 388 + return 389 389 + } 307 390 308 391 var result *xrpc.CreateRecordOutput 309 392 err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error {

+16

backend/internal/api/collections.go

reviewed

··· 54 54 55 55 record := xrpc.NewCollectionRecord(req.Name, req.Description, req.Icon) 56 56 57 57 + if err := record.Validate(); err != nil { 58 58 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 59 59 + return 60 60 + } 61 61 + 57 62 var result *xrpc.CreateRecordOutput 58 63 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 59 64 var createErr error ··· 115 120 } 116 121 117 122 record := xrpc.NewCollectionItemRecord(collectionURI, req.AnnotationURI, req.Position) 123 123 + 124 124 + if err := record.Validate(); err != nil { 125 125 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 126 126 + return 127 127 + } 118 128 119 129 var result *xrpc.CreateRecordOutput 120 130 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { ··· 368 378 } 369 379 370 380 record := xrpc.NewCollectionRecord(req.Name, req.Description, req.Icon) 381 381 + 382 382 + if err := record.Validate(); err != nil { 383 383 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 384 384 + return 385 385 + } 386 386 + 371 387 parts := strings.Split(uri, "/") 372 388 rkey := parts[len(parts)-1] 373 389

+221 -12

backend/internal/api/handler.go

reviewed

··· 8 8 "net/url" 9 9 "strconv" 10 10 "strings" 11 11 + "sync" 11 12 "time" 12 13 13 14 "github.com/go-chi/chi/v5" 14 15 15 16 "margin.at/internal/db" 17 17 + "margin.at/internal/xrpc" 16 18 ) 17 19 18 20 type Handler struct { ··· 57 59 r.Get("/collections/{collection}/items", collectionService.GetCollectionItems) 58 60 r.Delete("/collections/items", collectionService.RemoveCollectionItem) 59 61 r.Get("/collections/containing", collectionService.GetAnnotationCollections) 62 62 + r.Post("/sync", h.SyncAll) 60 63 61 64 r.Get("/targets", h.GetByTarget) 62 65 ··· 77 80 r.Delete("/keys/{id}", h.apiKeys.DeleteKey) 78 81 79 82 r.Post("/quick/bookmark", h.apiKeys.QuickBookmark) 80 80 - r.Post("/quick/annotation", h.apiKeys.QuickAnnotation) 81 81 - r.Post("/quick/highlight", h.apiKeys.QuickHighlight) 83 83 + r.Post("/quick/save", h.apiKeys.QuickSave) 82 84 }) 83 85 } 84 86 ··· 132 134 limit := parseIntParam(r, "limit", 50) 133 135 tag := r.URL.Query().Get("tag") 134 136 creator := r.URL.Query().Get("creator") 137 137 + 138 138 + viewerDID := h.getViewerDID(r) 139 139 + 140 140 + if viewerDID != "" && (creator == viewerDID || (creator == "" && tag == "")) { 141 141 + if creator == viewerDID { 142 142 + h.serveUserFeedFromPDS(w, r, viewerDID, tag, limit) 143 143 + return 144 144 + } 145 145 + } 135 146 136 147 var annotations []db.Annotation 137 148 var highlights []db.Highlight ··· 166 177 } 167 178 } 168 179 169 169 - viewerDID := h.getViewerDID(r) 170 180 authAnnos, _ := hydrateAnnotations(h.db, annotations, viewerDID) 171 181 authHighs, _ := hydrateHighlights(h.db, highlights, viewerDID) 172 182 authBooks, _ := hydrateBookmarks(h.db, bookmarks, viewerDID) ··· 187 197 feed = append(feed, ci) 188 198 } 189 199 190 190 - for i := 0; i < len(feed); i++ { 191 191 - for j := i + 1; j < len(feed); j++ { 192 192 - t1 := getCreatedAt(feed[i]) 193 193 - t2 := getCreatedAt(feed[j]) 194 194 - if t1.Before(t2) { 195 195 - feed[i], feed[j] = feed[j], feed[i] 200 200 + sortFeed(feed) 201 201 + 202 202 + if len(feed) > limit { 203 203 + feed = feed[:limit] 204 204 + } 205 205 + 206 206 + w.Header().Set("Content-Type", "application/json") 207 207 + json.NewEncoder(w).Encode(map[string]interface{}{ 208 208 + "@context": "http://www.w3.org/ns/anno.jsonld", 209 209 + "type": "Collection", 210 210 + "items": feed, 211 211 + "totalItems": len(feed), 212 212 + }) 213 213 + } 214 214 + 215 215 + func (h *Handler) serveUserFeedFromPDS(w http.ResponseWriter, r *http.Request, did, tag string, limit int) { 216 216 + var wg sync.WaitGroup 217 217 + var rawAnnos, rawHighs, rawBooks []interface{} 218 218 + var errAnnos, errHighs, errBooks error 219 219 + 220 220 + fetchLimit := limit * 2 221 221 + if fetchLimit < 50 { 222 222 + fetchLimit = 50 223 223 + } 224 224 + 225 225 + wg.Add(3) 226 226 + go func() { 227 227 + defer wg.Done() 228 228 + rawAnnos, errAnnos = h.FetchLatestUserRecords(r, did, xrpc.CollectionAnnotation, fetchLimit) 229 229 + }() 230 230 + go func() { 231 231 + defer wg.Done() 232 232 + rawHighs, errHighs = h.FetchLatestUserRecords(r, did, xrpc.CollectionHighlight, fetchLimit) 233 233 + }() 234 234 + go func() { 235 235 + defer wg.Done() 236 236 + rawBooks, errBooks = h.FetchLatestUserRecords(r, did, xrpc.CollectionBookmark, fetchLimit) 237 237 + }() 238 238 + wg.Wait() 239 239 + 240 240 + if errAnnos != nil { 241 241 + log.Printf("PDS Fetch Error (Annos): %v", errAnnos) 242 242 + } 243 243 + if errHighs != nil { 244 244 + log.Printf("PDS Fetch Error (Highs): %v", errHighs) 245 245 + } 246 246 + if errBooks != nil { 247 247 + log.Printf("PDS Fetch Error (Books): %v", errBooks) 248 248 + } 249 249 + 250 250 + var annotations []db.Annotation 251 251 + var highlights []db.Highlight 252 252 + var bookmarks []db.Bookmark 253 253 + 254 254 + for _, r := range rawAnnos { 255 255 + if a, ok := r.(*db.Annotation); ok { 256 256 + if tag == "" || containsTag(a.TagsJSON, tag) { 257 257 + annotations = append(annotations, *a) 258 258 + } 259 259 + } 260 260 + } 261 261 + for _, r := range rawHighs { 262 262 + if h, ok := r.(*db.Highlight); ok { 263 263 + if tag == "" || containsTag(h.TagsJSON, tag) { 264 264 + highlights = append(highlights, *h) 265 265 + } 266 266 + } 267 267 + } 268 268 + for _, r := range rawBooks { 269 269 + if b, ok := r.(*db.Bookmark); ok { 270 270 + if tag == "" || containsTag(b.TagsJSON, tag) { 271 271 + bookmarks = append(bookmarks, *b) 196 272 } 197 273 } 198 274 } 199 275 276 276 + go func() { 277 277 + for _, a := range annotations { 278 278 + h.db.CreateAnnotation(&a) 279 279 + } 280 280 + for _, hi := range highlights { 281 281 + h.db.CreateHighlight(&hi) 282 282 + } 283 283 + for _, b := range bookmarks { 284 284 + h.db.CreateBookmark(&b) 285 285 + } 286 286 + }() 287 287 + 288 288 + authAnnos, _ := hydrateAnnotations(h.db, annotations, did) 289 289 + authHighs, _ := hydrateHighlights(h.db, highlights, did) 290 290 + authBooks, _ := hydrateBookmarks(h.db, bookmarks, did) 291 291 + 292 292 + var feed []interface{} 293 293 + for _, a := range authAnnos { 294 294 + feed = append(feed, a) 295 295 + } 296 296 + for _, h := range authHighs { 297 297 + feed = append(feed, h) 298 298 + } 299 299 + for _, b := range authBooks { 300 300 + feed = append(feed, b) 301 301 + } 302 302 + 303 303 + sortFeed(feed) 304 304 + 200 305 if len(feed) > limit { 201 306 feed = feed[:limit] 202 307 } ··· 208 313 "items": feed, 209 314 "totalItems": len(feed), 210 315 }) 316 316 + 317 317 + } 318 318 + 319 319 + func containsTag(tagsJSON *string, tag string) bool { 320 320 + if tagsJSON == nil || *tagsJSON == "" { 321 321 + return false 322 322 + } 323 323 + var tags []string 324 324 + if err := json.Unmarshal([]byte(*tagsJSON), &tags); err != nil { 325 325 + return false 326 326 + } 327 327 + for _, t := range tags { 328 328 + if t == tag { 329 329 + return true 330 330 + } 331 331 + } 332 332 + return false 333 333 + } 334 334 + 335 335 + func sortFeed(feed []interface{}) { 336 336 + for i := 0; i < len(feed); i++ { 337 337 + for j := i + 1; j < len(feed); j++ { 338 338 + t1 := getCreatedAt(feed[i]) 339 339 + t2 := getCreatedAt(feed[j]) 340 340 + if t1.Before(t2) { 341 341 + feed[i], feed[j] = feed[j], feed[i] 342 342 + } 343 343 + } 344 344 + } 211 345 } 212 346 213 347 func getCreatedAt(item interface{}) time.Time { ··· 386 520 limit := parseIntParam(r, "limit", 50) 387 521 offset := parseIntParam(r, "offset", 0) 388 522 389 389 - annotations, err := h.db.GetAnnotationsByAuthor(did, limit, offset) 523 523 + var annotations []db.Annotation 524 524 + var err error 525 525 + 526 526 + viewerDID := h.getViewerDID(r) 527 527 + 528 528 + if offset == 0 && viewerDID != "" && did == viewerDID { 529 529 + raw, err := h.FetchLatestUserRecords(r, did, xrpc.CollectionAnnotation, limit) 530 530 + if err == nil { 531 531 + for _, r := range raw { 532 532 + if a, ok := r.(*db.Annotation); ok { 533 533 + annotations = append(annotations, *a) 534 534 + } 535 535 + } 536 536 + go func() { 537 537 + for _, a := range annotations { 538 538 + h.db.CreateAnnotation(&a) 539 539 + } 540 540 + }() 541 541 + } else { 542 542 + log.Printf("PDS Fetch Error (User Annos): %v", err) 543 543 + annotations, err = h.db.GetAnnotationsByAuthor(did, limit, offset) 544 544 + } 545 545 + } else { 546 546 + annotations, err = h.db.GetAnnotationsByAuthor(did, limit, offset) 547 547 + } 548 548 + 390 549 if err != nil { 391 550 http.Error(w, err.Error(), http.StatusInternalServerError) 392 551 return ··· 412 571 limit := parseIntParam(r, "limit", 50) 413 572 offset := parseIntParam(r, "offset", 0) 414 573 415 415 - highlights, err := h.db.GetHighlightsByAuthor(did, limit, offset) 574 574 + var highlights []db.Highlight 575 575 + var err error 576 576 + 577 577 + viewerDID := h.getViewerDID(r) 578 578 + 579 579 + if offset == 0 && viewerDID != "" && did == viewerDID { 580 580 + raw, err := h.FetchLatestUserRecords(r, did, xrpc.CollectionHighlight, limit) 581 581 + if err == nil { 582 582 + for _, r := range raw { 583 583 + if hi, ok := r.(*db.Highlight); ok { 584 584 + highlights = append(highlights, *hi) 585 585 + } 586 586 + } 587 587 + go func() { 588 588 + for _, hi := range highlights { 589 589 + h.db.CreateHighlight(&hi) 590 590 + } 591 591 + }() 592 592 + } else { 593 593 + log.Printf("PDS Fetch Error (User Highs): %v", err) 594 594 + highlights, err = h.db.GetHighlightsByAuthor(did, limit, offset) 595 595 + } 596 596 + } else { 597 597 + highlights, err = h.db.GetHighlightsByAuthor(did, limit, offset) 598 598 + } 599 599 + 416 600 if err != nil { 417 601 http.Error(w, err.Error(), http.StatusInternalServerError) 418 602 return ··· 438 622 limit := parseIntParam(r, "limit", 50) 439 623 offset := parseIntParam(r, "offset", 0) 440 624 441 441 - bookmarks, err := h.db.GetBookmarksByAuthor(did, limit, offset) 625 625 + var bookmarks []db.Bookmark 626 626 + var err error 627 627 + 628 628 + viewerDID := h.getViewerDID(r) 629 629 + 630 630 + if offset == 0 && viewerDID != "" && did == viewerDID { 631 631 + raw, err := h.FetchLatestUserRecords(r, did, xrpc.CollectionBookmark, limit) 632 632 + if err == nil { 633 633 + for _, r := range raw { 634 634 + if b, ok := r.(*db.Bookmark); ok { 635 635 + bookmarks = append(bookmarks, *b) 636 636 + } 637 637 + } 638 638 + go func() { 639 639 + for _, b := range bookmarks { 640 640 + h.db.CreateBookmark(&b) 641 641 + } 642 642 + }() 643 643 + } else { 644 644 + log.Printf("PDS Fetch Error (User Books): %v", err) 645 645 + bookmarks, err = h.db.GetBookmarksByAuthor(did, limit, offset) 646 646 + } 647 647 + } else { 648 648 + bookmarks, err = h.db.GetBookmarksByAuthor(did, limit, offset) 649 649 + } 650 650 + 442 651 if err != nil { 443 652 http.Error(w, err.Error(), http.StatusInternalServerError) 444 653 return

+226

backend/internal/api/pds.go

reviewed

··· 1 1 + package api 2 2 + 3 3 + import ( 4 4 + "encoding/json" 5 5 + "fmt" 6 6 + "net/http" 7 7 + "time" 8 8 + 9 9 + "margin.at/internal/db" 10 10 + "margin.at/internal/xrpc" 11 11 + ) 12 12 + 13 13 + func (h *Handler) FetchLatestUserRecords(r *http.Request, did string, collection string, limit int) ([]interface{}, error) { 14 14 + session, err := h.refresher.GetSessionWithAutoRefresh(r) 15 15 + if err != nil { 16 16 + return nil, err 17 17 + } 18 18 + 19 19 + var results []interface{} 20 20 + 21 21 + err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, _ string) error { 22 22 + url := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=%s&limit=%d", client.PDS, did, collection, limit) 23 23 + 24 24 + req, _ := http.NewRequestWithContext(r.Context(), "GET", url, nil) 25 25 + req.Header.Set("Authorization", "Bearer "+client.AccessToken) 26 26 + 27 27 + resp, err := http.DefaultClient.Do(req) 28 28 + if err != nil { 29 29 + return fmt.Errorf("failed to fetch %s: %w", collection, err) 30 30 + } 31 31 + defer resp.Body.Close() 32 32 + 33 33 + if resp.StatusCode != 200 { 34 34 + return fmt.Errorf("XRPC error %d", resp.StatusCode) 35 35 + } 36 36 + 37 37 + var output struct { 38 38 + Records []struct { 39 39 + URI string `json:"uri"` 40 40 + CID string `json:"cid"` 41 41 + Value json.RawMessage `json:"value"` 42 42 + } `json:"records"` 43 43 + Cursor string `json:"cursor"` 44 44 + } 45 45 + 46 46 + if err := json.NewDecoder(resp.Body).Decode(&output); err != nil { 47 47 + return err 48 48 + } 49 49 + 50 50 + for _, rec := range output.Records { 51 51 + parsed, err := parseRecord(did, collection, rec.URI, rec.CID, rec.Value) 52 52 + if err == nil && parsed != nil { 53 53 + results = append(results, parsed) 54 54 + } 55 55 + } 56 56 + return nil 57 57 + }) 58 58 + 59 59 + if err != nil { 60 60 + return nil, err 61 61 + } 62 62 + 63 63 + return results, nil 64 64 + } 65 65 + 66 66 + func parseRecord(did, collection, uri, cid string, value json.RawMessage) (interface{}, error) { 67 67 + cidPtr := &cid 68 68 + 69 69 + switch collection { 70 70 + case xrpc.CollectionAnnotation: 71 71 + var record xrpc.AnnotationRecord 72 72 + if err := json.Unmarshal(value, &record); err != nil { 73 73 + return nil, err 74 74 + } 75 75 + 76 76 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 77 77 + 78 78 + targetSource := record.Target.Source 79 79 + 80 80 + targetHash := record.Target.SourceHash 81 81 + if targetHash == "" && targetSource != "" { 82 82 + targetHash = db.HashURL(targetSource) 83 83 + } 84 84 + 85 85 + motivation := record.Motivation 86 86 + if motivation == "" { 87 87 + motivation = "commenting" 88 88 + } 89 89 + 90 90 + var bodyValuePtr, bodyFormatPtr, bodyURIPtr *string 91 91 + if record.Body != nil { 92 92 + if record.Body.Value != "" { 93 93 + val := record.Body.Value 94 94 + bodyValuePtr = &val 95 95 + } 96 96 + if record.Body.Format != "" { 97 97 + fmt := record.Body.Format 98 98 + bodyFormatPtr = &fmt 99 99 + } 100 100 + } 101 101 + 102 102 + var targetTitlePtr, selectorJSONPtr, tagsJSONPtr *string 103 103 + if record.Target.Title != "" { 104 104 + t := record.Target.Title 105 105 + targetTitlePtr = &t 106 106 + } 107 107 + if len(record.Target.Selector) > 0 { 108 108 + selectorStr := string(record.Target.Selector) 109 109 + selectorJSONPtr = &selectorStr 110 110 + } 111 111 + if len(record.Tags) > 0 { 112 112 + tagsBytes, _ := json.Marshal(record.Tags) 113 113 + tagsStr := string(tagsBytes) 114 114 + tagsJSONPtr = &tagsStr 115 115 + } 116 116 + 117 117 + return &db.Annotation{ 118 118 + URI: uri, 119 119 + AuthorDID: did, 120 120 + Motivation: motivation, 121 121 + BodyValue: bodyValuePtr, 122 122 + BodyFormat: bodyFormatPtr, 123 123 + BodyURI: bodyURIPtr, 124 124 + TargetSource: targetSource, 125 125 + TargetHash: targetHash, 126 126 + TargetTitle: targetTitlePtr, 127 127 + SelectorJSON: selectorJSONPtr, 128 128 + TagsJSON: tagsJSONPtr, 129 129 + CreatedAt: createdAt, 130 130 + IndexedAt: time.Now(), 131 131 + CID: cidPtr, 132 132 + }, nil 133 133 + 134 134 + case xrpc.CollectionHighlight: 135 135 + var record xrpc.HighlightRecord 136 136 + if err := json.Unmarshal(value, &record); err != nil { 137 137 + return nil, err 138 138 + } 139 139 + 140 140 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 141 141 + if createdAt.IsZero() { 142 142 + createdAt = time.Now() 143 143 + } 144 144 + 145 145 + targetHash := record.Target.SourceHash 146 146 + if targetHash == "" && record.Target.Source != "" { 147 147 + targetHash = db.HashURL(record.Target.Source) 148 148 + } 149 149 + 150 150 + var titlePtr, selectorJSONPtr, colorPtr, tagsJSONPtr *string 151 151 + if record.Target.Title != "" { 152 152 + t := record.Target.Title 153 153 + titlePtr = &t 154 154 + } 155 155 + if len(record.Target.Selector) > 0 { 156 156 + selectorStr := string(record.Target.Selector) 157 157 + selectorJSONPtr = &selectorStr 158 158 + } 159 159 + if record.Color != "" { 160 160 + c := record.Color 161 161 + colorPtr = &c 162 162 + } 163 163 + if len(record.Tags) > 0 { 164 164 + tagsBytes, _ := json.Marshal(record.Tags) 165 165 + tagsStr := string(tagsBytes) 166 166 + tagsJSONPtr = &tagsStr 167 167 + } 168 168 + 169 169 + return &db.Highlight{ 170 170 + URI: uri, 171 171 + AuthorDID: did, 172 172 + TargetSource: record.Target.Source, 173 173 + TargetHash: targetHash, 174 174 + TargetTitle: titlePtr, 175 175 + SelectorJSON: selectorJSONPtr, 176 176 + Color: colorPtr, 177 177 + TagsJSON: tagsJSONPtr, 178 178 + CreatedAt: createdAt, 179 179 + IndexedAt: time.Now(), 180 180 + CID: cidPtr, 181 181 + }, nil 182 182 + 183 183 + case xrpc.CollectionBookmark: 184 184 + var record xrpc.BookmarkRecord 185 185 + if err := json.Unmarshal(value, &record); err != nil { 186 186 + return nil, err 187 187 + } 188 188 + 189 189 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 190 190 + 191 191 + sourceHash := record.SourceHash 192 192 + if sourceHash == "" && record.Source != "" { 193 193 + sourceHash = db.HashURL(record.Source) 194 194 + } 195 195 + 196 196 + var titlePtr, descPtr, tagsJSONPtr *string 197 197 + if record.Title != "" { 198 198 + t := record.Title 199 199 + titlePtr = &t 200 200 + } 201 201 + if record.Description != "" { 202 202 + d := record.Description 203 203 + descPtr = &d 204 204 + } 205 205 + if len(record.Tags) > 0 { 206 206 + tagsBytes, _ := json.Marshal(record.Tags) 207 207 + tagsStr := string(tagsBytes) 208 208 + tagsJSONPtr = &tagsStr 209 209 + } 210 210 + 211 211 + return &db.Bookmark{ 212 212 + URI: uri, 213 213 + AuthorDID: did, 214 214 + Source: record.Source, 215 215 + SourceHash: sourceHash, 216 216 + Title: titlePtr, 217 217 + Description: descPtr, 218 218 + TagsJSON: tagsJSONPtr, 219 219 + CreatedAt: createdAt, 220 220 + IndexedAt: time.Now(), 221 221 + CID: cidPtr, 222 222 + }, nil 223 223 + } 224 224 + 225 225 + return nil, nil 226 226 + }

+351

backend/internal/api/sync.go

reviewed

··· 1 1 + package api 2 2 + 3 3 + import ( 4 4 + "encoding/json" 5 5 + "fmt" 6 6 + "io" 7 7 + "net/http" 8 8 + "time" 9 9 + 10 10 + "margin.at/internal/db" 11 11 + "margin.at/internal/xrpc" 12 12 + ) 13 13 + 14 14 + func (h *Handler) SyncAll(w http.ResponseWriter, r *http.Request) { 15 15 + session, err := h.refresher.GetSessionWithAutoRefresh(r) 16 16 + if err != nil { 17 17 + http.Error(w, err.Error(), http.StatusUnauthorized) 18 18 + return 19 19 + } 20 20 + 21 21 + collections := []string{ 22 22 + xrpc.CollectionAnnotation, 23 23 + xrpc.CollectionHighlight, 24 24 + xrpc.CollectionBookmark, 25 25 + xrpc.CollectionReply, 26 26 + xrpc.CollectionLike, 27 27 + xrpc.CollectionCollection, 28 28 + xrpc.CollectionCollectionItem, 29 29 + } 30 30 + 31 31 + results := make(map[string]string) 32 32 + 33 33 + err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 34 34 + for _, collectionNSID := range collections { 35 35 + count := 0 36 36 + cursor := "" 37 37 + fetchedURIs := make(map[string]bool) 38 38 + 39 39 + for { 40 40 + url := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=%s&limit=100", client.PDS, did, collectionNSID) 41 41 + if cursor != "" { 42 42 + url += "&cursor=" + cursor 43 43 + } 44 44 + 45 45 + req, _ := http.NewRequestWithContext(r.Context(), "GET", url, nil) 46 46 + req.Header.Set("Authorization", "Bearer "+client.AccessToken) 47 47 + 48 48 + resp, err := http.DefaultClient.Do(req) 49 49 + if err != nil { 50 50 + return fmt.Errorf("failed to fetch %s: %w", collectionNSID, err) 51 51 + } 52 52 + defer resp.Body.Close() 53 53 + 54 54 + if resp.StatusCode != 200 { 55 55 + body, _ := io.ReadAll(resp.Body) 56 56 + results[collectionNSID] = fmt.Sprintf("error: %s", string(body)) 57 57 + break 58 58 + } 59 59 + 60 60 + var output struct { 61 61 + Records []struct { 62 62 + URI string `json:"uri"` 63 63 + CID string `json:"cid"` 64 64 + Value json.RawMessage `json:"value"` 65 65 + } `json:"records"` 66 66 + Cursor string `json:"cursor"` 67 67 + } 68 68 + 69 69 + if err := json.NewDecoder(resp.Body).Decode(&output); err != nil { 70 70 + return err 71 71 + } 72 72 + 73 73 + for _, rec := range output.Records { 74 74 + err := h.upsertRecord(did, collectionNSID, rec.URI, rec.CID, rec.Value) 75 75 + if err != nil { 76 76 + fmt.Printf("Error upserting %s: %v\n", rec.URI, err) 77 77 + } else { 78 78 + count++ 79 79 + fetchedURIs[rec.URI] = true 80 80 + } 81 81 + } 82 82 + 83 83 + if output.Cursor == "" { 84 84 + break 85 85 + } 86 86 + cursor = output.Cursor 87 87 + } 88 88 + 89 89 + deletedCount := 0 90 90 + if results[collectionNSID] == "" { 91 91 + var localURIs []string 92 92 + var err error 93 93 + 94 94 + switch collectionNSID { 95 95 + case xrpc.CollectionAnnotation: 96 96 + localURIs, err = h.db.GetAnnotationURIs(did) 97 97 + case xrpc.CollectionHighlight: 98 98 + localURIs, err = h.db.GetHighlightURIs(did) 99 99 + case xrpc.CollectionBookmark: 100 100 + localURIs, err = h.db.GetBookmarkURIs(did) 101 101 + } 102 102 + 103 103 + if err == nil { 104 104 + for _, uri := range localURIs { 105 105 + if !fetchedURIs[uri] { 106 106 + switch collectionNSID { 107 107 + case xrpc.CollectionAnnotation: 108 108 + _ = h.db.DeleteAnnotation(uri) 109 109 + case xrpc.CollectionHighlight: 110 110 + _ = h.db.DeleteHighlight(uri) 111 111 + case xrpc.CollectionBookmark: 112 112 + _ = h.db.DeleteBookmark(uri) 113 113 + } 114 114 + deletedCount++ 115 115 + } 116 116 + } 117 117 + } 118 118 + } 119 119 + 120 120 + if results[collectionNSID] == "" { 121 121 + results[collectionNSID] = fmt.Sprintf("synced %d records, deleted %d stale", count, deletedCount) 122 122 + } 123 123 + } 124 124 + return nil 125 125 + }) 126 126 + 127 127 + if err != nil { 128 128 + http.Error(w, "Sync failed: "+err.Error(), http.StatusInternalServerError) 129 129 + return 130 130 + } 131 131 + 132 132 + w.WriteHeader(http.StatusOK) 133 133 + json.NewEncoder(w).Encode(results) 134 134 + } 135 135 + 136 136 + func strPtr(s string) *string { 137 137 + if s == "" { 138 138 + return nil 139 139 + } 140 140 + return &s 141 141 + } 142 142 + 143 143 + func (h *Handler) upsertRecord(did, collection, uri, cid string, value json.RawMessage) error { 144 144 + cidPtr := strPtr(cid) 145 145 + switch collection { 146 146 + case xrpc.CollectionAnnotation: 147 147 + var record xrpc.AnnotationRecord 148 148 + if err := json.Unmarshal(value, &record); err != nil { 149 149 + return err 150 150 + } 151 151 + 152 152 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 153 153 + 154 154 + targetSource := record.Target.Source 155 155 + if targetSource == "" { 156 156 + 157 157 + } 158 158 + 159 159 + targetHash := record.Target.SourceHash 160 160 + if targetHash == "" && targetSource != "" { 161 161 + targetHash = db.HashURL(targetSource) 162 162 + } 163 163 + 164 164 + motivation := record.Motivation 165 165 + if motivation == "" { 166 166 + motivation = "commenting" 167 167 + } 168 168 + 169 169 + var bodyValuePtr, bodyFormatPtr, bodyURIPtr, targetTitlePtr, selectorJSONPtr, tagsJSONPtr *string 170 170 + if record.Body != nil { 171 171 + if record.Body.Value != "" { 172 172 + val := record.Body.Value 173 173 + bodyValuePtr = &val 174 174 + } 175 175 + if record.Body.Format != "" { 176 176 + fmt := record.Body.Format 177 177 + bodyFormatPtr = &fmt 178 178 + } 179 179 + } 180 180 + if record.Target.Title != "" { 181 181 + t := record.Target.Title 182 182 + targetTitlePtr = &t 183 183 + } 184 184 + if len(record.Target.Selector) > 0 { 185 185 + selectorStr := string(record.Target.Selector) 186 186 + selectorJSONPtr = &selectorStr 187 187 + } 188 188 + if len(record.Tags) > 0 { 189 189 + tagsBytes, _ := json.Marshal(record.Tags) 190 190 + tagsStr := string(tagsBytes) 191 191 + tagsJSONPtr = &tagsStr 192 192 + } 193 193 + 194 194 + return h.db.CreateAnnotation(&db.Annotation{ 195 195 + URI: uri, 196 196 + AuthorDID: did, 197 197 + Motivation: motivation, 198 198 + BodyValue: bodyValuePtr, 199 199 + BodyFormat: bodyFormatPtr, 200 200 + BodyURI: bodyURIPtr, 201 201 + TargetSource: targetSource, 202 202 + TargetHash: targetHash, 203 203 + TargetTitle: targetTitlePtr, 204 204 + SelectorJSON: selectorJSONPtr, 205 205 + TagsJSON: tagsJSONPtr, 206 206 + CreatedAt: createdAt, 207 207 + IndexedAt: time.Now(), 208 208 + CID: cidPtr, 209 209 + }) 210 210 + 211 211 + case xrpc.CollectionHighlight: 212 212 + var record xrpc.HighlightRecord 213 213 + if err := json.Unmarshal(value, &record); err != nil { 214 214 + return err 215 215 + } 216 216 + 217 217 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 218 218 + if createdAt.IsZero() { 219 219 + createdAt = time.Now() 220 220 + } 221 221 + 222 222 + targetHash := record.Target.SourceHash 223 223 + if targetHash == "" && record.Target.Source != "" { 224 224 + targetHash = db.HashURL(record.Target.Source) 225 225 + } 226 226 + 227 227 + var titlePtr, selectorJSONPtr, colorPtr, tagsJSONPtr *string 228 228 + if record.Target.Title != "" { 229 229 + t := record.Target.Title 230 230 + titlePtr = &t 231 231 + } 232 232 + if len(record.Target.Selector) > 0 { 233 233 + selectorStr := string(record.Target.Selector) 234 234 + selectorJSONPtr = &selectorStr 235 235 + } 236 236 + if record.Color != "" { 237 237 + c := record.Color 238 238 + colorPtr = &c 239 239 + } 240 240 + if len(record.Tags) > 0 { 241 241 + tagsBytes, _ := json.Marshal(record.Tags) 242 242 + tagsStr := string(tagsBytes) 243 243 + tagsJSONPtr = &tagsStr 244 244 + } 245 245 + 246 246 + return h.db.CreateHighlight(&db.Highlight{ 247 247 + URI: uri, 248 248 + AuthorDID: did, 249 249 + TargetSource: record.Target.Source, 250 250 + TargetHash: targetHash, 251 251 + TargetTitle: titlePtr, 252 252 + SelectorJSON: selectorJSONPtr, 253 253 + Color: colorPtr, 254 254 + TagsJSON: tagsJSONPtr, 255 255 + CreatedAt: createdAt, 256 256 + IndexedAt: time.Now(), 257 257 + CID: cidPtr, 258 258 + }) 259 259 + 260 260 + case xrpc.CollectionBookmark: 261 261 + var record xrpc.BookmarkRecord 262 262 + if err := json.Unmarshal(value, &record); err != nil { 263 263 + return err 264 264 + } 265 265 + 266 266 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 267 267 + 268 268 + sourceHash := record.SourceHash 269 269 + if sourceHash == "" && record.Source != "" { 270 270 + sourceHash = db.HashURL(record.Source) 271 271 + } 272 272 + 273 273 + var titlePtr, descPtr, tagsJSONPtr *string 274 274 + if record.Title != "" { 275 275 + t := record.Title 276 276 + titlePtr = &t 277 277 + } 278 278 + if record.Description != "" { 279 279 + d := record.Description 280 280 + descPtr = &d 281 281 + } 282 282 + if len(record.Tags) > 0 { 283 283 + tagsBytes, _ := json.Marshal(record.Tags) 284 284 + tagsStr := string(tagsBytes) 285 285 + tagsJSONPtr = &tagsStr 286 286 + } 287 287 + 288 288 + return h.db.CreateBookmark(&db.Bookmark{ 289 289 + URI: uri, 290 290 + AuthorDID: did, 291 291 + Source: record.Source, 292 292 + SourceHash: sourceHash, 293 293 + Title: titlePtr, 294 294 + Description: descPtr, 295 295 + TagsJSON: tagsJSONPtr, 296 296 + CreatedAt: createdAt, 297 297 + IndexedAt: time.Now(), 298 298 + CID: cidPtr, 299 299 + }) 300 300 + 301 301 + case xrpc.CollectionCollection: 302 302 + var record xrpc.CollectionRecord 303 303 + if err := json.Unmarshal(value, &record); err != nil { 304 304 + return err 305 305 + } 306 306 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 307 307 + 308 308 + var descPtr, iconPtr *string 309 309 + if record.Description != "" { 310 310 + d := record.Description 311 311 + descPtr = &d 312 312 + } 313 313 + if record.Icon != "" { 314 314 + i := record.Icon 315 315 + iconPtr = &i 316 316 + } 317 317 + 318 318 + return h.db.CreateCollection(&db.Collection{ 319 319 + URI: uri, 320 320 + AuthorDID: did, 321 321 + Name: record.Name, 322 322 + Description: descPtr, 323 323 + Icon: iconPtr, 324 324 + CreatedAt: createdAt, 325 325 + IndexedAt: time.Now(), 326 326 + }) 327 327 + 328 328 + case xrpc.CollectionCollectionItem: 329 329 + var record xrpc.CollectionItemRecord 330 330 + if err := json.Unmarshal(value, &record); err != nil { 331 331 + return err 332 332 + } 333 333 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 334 334 + 335 335 + return h.db.AddToCollection(&db.CollectionItem{ 336 336 + URI: uri, 337 337 + AuthorDID: did, 338 338 + CollectionURI: record.Collection, 339 339 + AnnotationURI: record.Annotation, 340 340 + Position: record.Position, 341 341 + CreatedAt: createdAt, 342 342 + IndexedAt: time.Now(), 343 343 + }) 344 344 + 345 345 + case xrpc.CollectionReply: 346 346 + return nil 347 347 + case xrpc.CollectionLike: 348 348 + return nil 349 349 + } 350 350 + return nil 351 351 + }

+20

backend/internal/db/queries_annotations.go

reviewed

··· 170 170 171 171 return scanAnnotations(rows) 172 172 } 173 173 + 174 174 + func (db *DB) GetAnnotationURIs(authorDID string) ([]string, error) { 175 175 + rows, err := db.Query(db.Rebind(` 176 176 + SELECT uri FROM annotations WHERE author_did = ? 177 177 + `), authorDID) 178 178 + if err != nil { 179 179 + return nil, err 180 180 + } 181 181 + defer rows.Close() 182 182 + 183 183 + var uris []string 184 184 + for rows.Next() { 185 185 + var uri string 186 186 + if err := rows.Scan(&uri); err != nil { 187 187 + return nil, err 188 188 + } 189 189 + uris = append(uris, uri) 190 190 + } 191 191 + return uris, nil 192 192 + }

+20

backend/internal/db/queries_bookmarks.go

reviewed

··· 174 174 } 175 175 return bookmarks, nil 176 176 } 177 177 + 178 178 + func (db *DB) GetBookmarkURIs(authorDID string) ([]string, error) { 179 179 + rows, err := db.Query(db.Rebind(` 180 180 + SELECT uri FROM bookmarks WHERE author_did = ? 181 181 + `), authorDID) 182 182 + if err != nil { 183 183 + return nil, err 184 184 + } 185 185 + defer rows.Close() 186 186 + 187 187 + var uris []string 188 188 + for rows.Next() { 189 189 + var uri string 190 190 + if err := rows.Scan(&uri); err != nil { 191 191 + return nil, err 192 192 + } 193 193 + uris = append(uris, uri) 194 194 + } 195 195 + return uris, nil 196 196 + }

+23

backend/internal/db/queries_collections.go

reviewed

··· 118 118 return items, nil 119 119 } 120 120 121 121 + func (db *DB) GetCollectionItemsByAuthor(authorDID string) ([]CollectionItem, error) { 122 122 + rows, err := db.Query(db.Rebind(` 123 123 + SELECT uri, author_did, collection_uri, annotation_uri, position, created_at, indexed_at 124 124 + FROM collection_items 125 125 + WHERE author_did = ? 126 126 + ORDER BY created_at DESC 127 127 + `), authorDID) 128 128 + if err != nil { 129 129 + return nil, err 130 130 + } 131 131 + defer rows.Close() 132 132 + 133 133 + var items []CollectionItem 134 134 + for rows.Next() { 135 135 + var item CollectionItem 136 136 + if err := rows.Scan(&item.URI, &item.AuthorDID, &item.CollectionURI, &item.AnnotationURI, &item.Position, &item.CreatedAt, &item.IndexedAt); err != nil { 137 137 + return nil, err 138 138 + } 139 139 + items = append(items, item) 140 140 + } 141 141 + return items, nil 142 142 + } 143 143 + 121 144 func (db *DB) GetCollectionURIsForAnnotation(annotationURI string) ([]string, error) { 122 145 rows, err := db.Query(db.Rebind(` 123 146 SELECT collection_uri FROM collection_items WHERE annotation_uri = ?

+20

backend/internal/db/queries_highlights.go

reviewed

··· 199 199 } 200 200 return highlights, nil 201 201 } 202 202 + 203 203 + func (db *DB) GetHighlightURIs(authorDID string) ([]string, error) { 204 204 + rows, err := db.Query(db.Rebind(` 205 205 + SELECT uri FROM highlights WHERE author_did = ? 206 206 + `), authorDID) 207 207 + if err != nil { 208 208 + return nil, err 209 209 + } 210 210 + defer rows.Close() 211 211 + 212 212 + var uris []string 213 213 + for rows.Next() { 214 214 + var uri string 215 215 + if err := rows.Scan(&uri); err != nil { 216 216 + return nil, err 217 217 + } 218 218 + uris = append(uris, uri) 219 219 + } 220 220 + return uris, nil 221 221 + }

+23

backend/internal/db/queries_likes.go

reviewed

··· 14 14 return err 15 15 } 16 16 17 17 + func (db *DB) GetLikesByAuthor(authorDID string) ([]Like, error) { 18 18 + rows, err := db.Query(db.Rebind(` 19 19 + SELECT uri, author_did, subject_uri, created_at, indexed_at 20 20 + FROM likes 21 21 + WHERE author_did = ? 22 22 + ORDER BY created_at DESC 23 23 + `), authorDID) 24 24 + if err != nil { 25 25 + return nil, err 26 26 + } 27 27 + defer rows.Close() 28 28 + 29 29 + var likes []Like 30 30 + for rows.Next() { 31 31 + var l Like 32 32 + if err := rows.Scan(&l.URI, &l.AuthorDID, &l.SubjectURI, &l.CreatedAt, &l.IndexedAt); err != nil { 33 33 + return nil, err 34 34 + } 35 35 + likes = append(likes, l) 36 36 + } 37 37 + return likes, nil 38 38 + } 39 39 + 17 40 func (db *DB) GetLikeCount(subjectURI string) (int, error) { 18 41 var count int 19 42 err := db.QueryRow(db.Rebind(`SELECT COUNT(*) FROM likes WHERE subject_uri = ?`), subjectURI).Scan(&count)

+185 -15

backend/internal/xrpc/records.go

reviewed

··· 1 1 package xrpc 2 2 3 3 - import "time" 3 3 + import ( 4 4 + "encoding/json" 5 5 + "fmt" 6 6 + "time" 7 7 + "unicode/utf8" 8 8 + ) 4 9 5 10 const ( 6 11 CollectionAnnotation = "at.margin.annotation" ··· 12 17 CollectionCollectionItem = "at.margin.collectionItem" 13 18 ) 14 19 20 20 + const ( 21 21 + SelectorTypeQuote = "TextQuoteSelector" 22 22 + SelectorTypePosition = "TextPositionSelector" 23 23 + ) 24 24 + 25 25 + type Selector struct { 26 26 + Type string `json:"type"` 27 27 + } 28 28 + 29 29 + type TextQuoteSelector struct { 30 30 + Type string `json:"type"` 31 31 + Exact string `json:"exact"` 32 32 + Prefix string `json:"prefix,omitempty"` 33 33 + Suffix string `json:"suffix,omitempty"` 34 34 + } 35 35 + 36 36 + func (s *TextQuoteSelector) Validate() error { 37 37 + if s.Type != SelectorTypeQuote { 38 38 + return fmt.Errorf("invalid selector type: %s", s.Type) 39 39 + } 40 40 + if len(s.Exact) > 5000 { 41 41 + return fmt.Errorf("exact text too long: %d > 5000", len(s.Exact)) 42 42 + } 43 43 + if len(s.Prefix) > 500 { 44 44 + return fmt.Errorf("prefix too long: %d > 500", len(s.Prefix)) 45 45 + } 46 46 + if len(s.Suffix) > 500 { 47 47 + return fmt.Errorf("suffix too long: %d > 500", len(s.Suffix)) 48 48 + } 49 49 + return nil 50 50 + } 51 51 + 52 52 + type TextPositionSelector struct { 53 53 + Type string `json:"type"` 54 54 + Start int `json:"start"` 55 55 + End int `json:"end"` 56 56 + } 57 57 + 58 58 + func (s *TextPositionSelector) Validate() error { 59 59 + if s.Type != SelectorTypePosition { 60 60 + return fmt.Errorf("invalid selector type: %s", s.Type) 61 61 + } 62 62 + if s.Start < 0 { 63 63 + return fmt.Errorf("start position cannot be negative") 64 64 + } 65 65 + if s.End < s.Start { 66 66 + return fmt.Errorf("end position cannot be before start") 67 67 + } 68 68 + return nil 69 69 + } 70 70 + 15 71 type AnnotationRecord struct { 16 72 Type string `json:"$type"` 17 73 Motivation string `json:"motivation,omitempty"` ··· 27 83 } 28 84 29 85 type AnnotationTarget struct { 30 30 - Source string `json:"source"` 31 31 - SourceHash string `json:"sourceHash"` 32 32 - Title string `json:"title,omitempty"` 33 33 - Selector interface{} `json:"selector,omitempty"` 86 86 + Source string `json:"source"` 87 87 + SourceHash string `json:"sourceHash"` 88 88 + Title string `json:"title,omitempty"` 89 89 + Selector json.RawMessage `json:"selector,omitempty"` 34 90 } 35 91 36 36 - type TextQuoteSelector struct { 37 37 - Type string `json:"type"` 38 38 - Exact string `json:"exact"` 39 39 - Prefix string `json:"prefix,omitempty"` 40 40 - Suffix string `json:"suffix,omitempty"` 92 92 + func (r *AnnotationRecord) Validate() error { 93 93 + if r.Target.Source == "" { 94 94 + return fmt.Errorf("target source is required") 95 95 + } 96 96 + if r.Body != nil { 97 97 + if len(r.Body.Value) > 10000 { 98 98 + return fmt.Errorf("body too long: %d > 10000", len(r.Body.Value)) 99 99 + } 100 100 + if utf8.RuneCountInString(r.Body.Value) > 3000 { 101 101 + return fmt.Errorf("body too long (graphemes): %d > 3000", utf8.RuneCountInString(r.Body.Value)) 102 102 + } 103 103 + } 104 104 + if len(r.Tags) > 10 { 105 105 + return fmt.Errorf("too many tags: %d > 10", len(r.Tags)) 106 106 + } 107 107 + for _, tag := range r.Tags { 108 108 + if len(tag) > 64 { 109 109 + return fmt.Errorf("tag too long: %s", tag) 110 110 + } 111 111 + } 112 112 + 113 113 + if len(r.Target.Selector) > 0 { 114 114 + var typeCheck Selector 115 115 + if err := json.Unmarshal(r.Target.Selector, &typeCheck); err != nil { 116 116 + return fmt.Errorf("invalid selector format") 117 117 + } 118 118 + 119 119 + switch typeCheck.Type { 120 120 + case SelectorTypeQuote: 121 121 + var s TextQuoteSelector 122 122 + if err := json.Unmarshal(r.Target.Selector, &s); err != nil { 123 123 + return err 124 124 + } 125 125 + return s.Validate() 126 126 + case SelectorTypePosition: 127 127 + var s TextPositionSelector 128 128 + if err := json.Unmarshal(r.Target.Selector, &s); err != nil { 129 129 + return err 130 130 + } 131 131 + return s.Validate() 132 132 + } 133 133 + } 134 134 + 135 135 + return nil 41 136 } 42 137 43 138 func NewAnnotationRecord(url, urlHash, text string, selector interface{}, title string) *AnnotationRecord { ··· 45 140 } 46 141 47 142 func NewAnnotationRecordWithMotivation(url, urlHash, text string, selector interface{}, title string, motivation string) *AnnotationRecord { 143 143 + var selectorJSON json.RawMessage 144 144 + if selector != nil { 145 145 + b, _ := json.Marshal(selector) 146 146 + selectorJSON = b 147 147 + } 148 148 + 48 149 record := &AnnotationRecord{ 49 150 Type: CollectionAnnotation, 50 151 Motivation: motivation, ··· 52 153 Source: url, 53 154 SourceHash: urlHash, 54 155 Title: title, 156 156 + Selector: selectorJSON, 55 157 }, 56 158 CreatedAt: time.Now().UTC().Format(time.RFC3339), 57 159 } ··· 63 165 } 64 166 } 65 167 66 66 - if selector != nil { 67 67 - record.Target.Selector = selector 68 68 - } 69 69 - 70 168 return record 71 169 } 72 170 ··· 78 176 CreatedAt string `json:"createdAt"` 79 177 } 80 178 179 179 + func (r *HighlightRecord) Validate() error { 180 180 + if r.Target.Source == "" { 181 181 + return fmt.Errorf("target source is required") 182 182 + } 183 183 + if len(r.Tags) > 10 { 184 184 + return fmt.Errorf("too many tags: %d", len(r.Tags)) 185 185 + } 186 186 + if len(r.Color) > 20 { 187 187 + return fmt.Errorf("color too long") 188 188 + } 189 189 + return nil 190 190 + } 191 191 + 81 192 func NewHighlightRecord(url, urlHash string, selector interface{}, color string, tags []string) *HighlightRecord { 193 193 + var selectorJSON json.RawMessage 194 194 + if selector != nil { 195 195 + b, _ := json.Marshal(selector) 196 196 + selectorJSON = b 197 197 + } 198 198 + 82 199 return &HighlightRecord{ 83 200 Type: CollectionHighlight, 84 201 Target: AnnotationTarget{ 85 202 Source: url, 86 203 SourceHash: urlHash, 87 87 - Selector: selector, 204 204 + Selector: selectorJSON, 88 205 }, 89 206 Color: color, 90 207 Tags: tags, ··· 106 223 CreatedAt string `json:"createdAt"` 107 224 } 108 225 226 226 + func (r *ReplyRecord) Validate() error { 227 227 + if r.Text == "" { 228 228 + return fmt.Errorf("text is required") 229 229 + } 230 230 + if len(r.Text) > 2000 { 231 231 + return fmt.Errorf("reply text too long") 232 232 + } 233 233 + return nil 234 234 + } 235 235 + 109 236 func NewReplyRecord(parentURI, parentCID, rootURI, rootCID, text string) *ReplyRecord { 110 237 return &ReplyRecord{ 111 238 Type: CollectionReply, ··· 128 255 CreatedAt string `json:"createdAt"` 129 256 } 130 257 258 258 + func (r *LikeRecord) Validate() error { 259 259 + if r.Subject.URI == "" || r.Subject.CID == "" { 260 260 + return fmt.Errorf("invalid subject") 261 261 + } 262 262 + return nil 263 263 + } 264 264 + 131 265 func NewLikeRecord(subjectURI, subjectCID string) *LikeRecord { 132 266 return &LikeRecord{ 133 267 Type: CollectionLike, ··· 146 280 CreatedAt string `json:"createdAt"` 147 281 } 148 282 283 283 + func (r *BookmarkRecord) Validate() error { 284 284 + if r.Source == "" { 285 285 + return fmt.Errorf("source is required") 286 286 + } 287 287 + if len(r.Title) > 500 { 288 288 + return fmt.Errorf("title too long") 289 289 + } 290 290 + if len(r.Description) > 1000 { 291 291 + return fmt.Errorf("description too long") 292 292 + } 293 293 + if len(r.Tags) > 10 { 294 294 + return fmt.Errorf("too many tags") 295 295 + } 296 296 + return nil 297 297 + } 298 298 + 149 299 func NewBookmarkRecord(url, urlHash, title, description string) *BookmarkRecord { 150 300 return &BookmarkRecord{ 151 301 Type: CollectionBookmark, ··· 165 315 CreatedAt string `json:"createdAt"` 166 316 } 167 317 318 318 + func (r *CollectionRecord) Validate() error { 319 319 + if r.Name == "" { 320 320 + return fmt.Errorf("name is required") 321 321 + } 322 322 + if len(r.Name) > 100 { 323 323 + return fmt.Errorf("name too long") 324 324 + } 325 325 + if len(r.Description) > 500 { 326 326 + return fmt.Errorf("description too long") 327 327 + } 328 328 + return nil 329 329 + } 330 330 + 168 331 func NewCollectionRecord(name, description, icon string) *CollectionRecord { 169 332 return &CollectionRecord{ 170 333 Type: CollectionCollection, ··· 181 344 Annotation string `json:"annotation"` 182 345 Position int `json:"position,omitempty"` 183 346 CreatedAt string `json:"createdAt"` 347 347 + } 348 348 + 349 349 + func (r *CollectionItemRecord) Validate() error { 350 350 + if r.Collection == "" || r.Annotation == "" { 351 351 + return fmt.Errorf("collection and annotation URIs required") 352 352 + } 353 353 + return nil 184 354 } 185 355 186 356 func NewCollectionItemRecord(collection, annotation string, position int) *CollectionItemRecord {

+10 -1

web/src/App.jsx

reviewed

··· 1 1 import { Routes, Route } from "react-router-dom"; 2 2 - import { AuthProvider } from "./context/AuthContext"; 2 2 + import { useEffect } from "react"; 3 3 + import { AuthProvider, useAuth } from "./context/AuthContext"; 3 4 import Sidebar from "./components/Sidebar"; 4 5 import RightSidebar from "./components/RightSidebar"; 5 6 import MobileNav from "./components/MobileNav"; ··· 19 20 import ScrollToTop from "./components/ScrollToTop"; 20 21 21 22 function AppContent() { 23 23 + const { user } = useAuth(); 24 24 + 25 25 + useEffect(() => { 26 26 + if (user) { 27 27 + fetch("/api/sync", { method: "POST" }).catch(console.error); 28 28 + } 29 29 + }, [user]); 30 30 + 22 31 return ( 23 32 <div className="layout"> 24 33 <ScrollToTop />