A container registry that uses the AT Protocol for manifest storage and S3 for blob storage.
better handling for io.atcr.repo.page
+231
-158
+1
-1
lexicons/io/atcr/repo/page.json
+1
-1
lexicons/io/atcr/repo/page.json
+6
-25
pkg/appview/middleware/registry.go
+6
-25
pkg/appview/middleware/registry.go
···
2
2
3
3
import (
4
4
"context"
5
-
"encoding/json"
6
5
"fmt"
7
6
"log/slog"
8
7
"net/http"
···
16
15
"github.com/distribution/distribution/v3/registry/storage/driver"
17
16
"github.com/distribution/reference"
18
17
18
+
"atcr.io/pkg/appview/readme"
19
19
"atcr.io/pkg/appview/storage"
20
20
"atcr.io/pkg/atproto"
21
21
"atcr.io/pkg/auth"
···
208
208
database storage.DatabaseMetrics // Metrics database (copied from global on init)
209
209
authorizer auth.HoldAuthorizer // Hold authorization (copied from global on init)
210
210
validationCache *validationCache // Request-level service token cache
211
+
readmeFetcher *readme.Fetcher // README fetcher for repo pages
211
212
}
212
213
213
214
// initATProtoResolver initializes the name resolution middleware
···
242
243
database: globalDatabase,
243
244
authorizer: globalAuthorizer,
244
245
validationCache: newValidationCache(),
246
+
readmeFetcher: readme.NewFetcher(),
245
247
}, nil
246
248
}
247
249
···
458
460
Database: nr.database,
459
461
Authorizer: nr.authorizer,
460
462
Refresher: nr.refresher,
463
+
ReadmeFetcher: nr.readmeFetcher,
461
464
}
462
465
463
466
return storage.NewRoutingRepository(repo, registryCtx), nil
···
481
484
// findHoldDID determines which hold DID to use for blob storage
482
485
// Priority order:
483
486
// 1. User's sailor profile defaultHold (if set)
484
-
// 2. User's own hold record (io.atcr.hold)
485
-
// 3. AppView's default hold DID
487
+
// 2. AppView's default hold DID
486
488
// Returns a hold DID (e.g., "did:web:hold01.atcr.io"), or empty string if none configured
487
489
func (nr *NamespaceResolver) findHoldDID(ctx context.Context, did, pdsEndpoint string) string {
488
490
// Create ATProto client (without auth - reading public records)
···
508
510
return profile.DefaultHold
509
511
}
510
512
511
-
// Profile doesn't exist or defaultHold is null/empty
512
-
// Check for user's own hold records
513
-
records, err := client.ListRecords(ctx, atproto.HoldCollection, 10)
514
-
if err != nil {
515
-
// Failed to query holds, use default
516
-
return nr.defaultHoldDID
517
-
}
518
-
519
-
// Find the first hold record
520
-
for _, record := range records {
521
-
var holdRecord atproto.HoldRecord
522
-
if err := json.Unmarshal(record.Value, &holdRecord); err != nil {
523
-
continue
524
-
}
525
-
526
-
// Return the endpoint from the first hold (normalize to DID if URL)
527
-
if holdRecord.Endpoint != "" {
528
-
return atproto.ResolveHoldDIDFromURL(holdRecord.Endpoint)
529
-
}
530
-
}
531
-
532
-
// No profile defaultHold and no own hold records - use AppView default
513
+
// No profile defaultHold - use AppView default
533
514
return nr.defaultHoldDID
534
515
}
535
516
-54
pkg/appview/middleware/registry_test.go
-54
pkg/appview/middleware/registry_test.go
···
199
199
assert.Equal(t, "did:web:user.hold.io", holdDID, "should use sailor profile's defaultHold")
200
200
}
201
201
202
-
// TestFindHoldDID_LegacyHoldRecords tests legacy hold record discovery
203
-
func TestFindHoldDID_LegacyHoldRecords(t *testing.T) {
204
-
// Start a mock PDS server that returns hold records
205
-
mockPDS := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
206
-
if r.URL.Path == "/xrpc/com.atproto.repo.getRecord" {
207
-
// Profile not found
208
-
w.WriteHeader(http.StatusNotFound)
209
-
return
210
-
}
211
-
if r.URL.Path == "/xrpc/com.atproto.repo.listRecords" {
212
-
// Return hold record
213
-
holdRecord := atproto.NewHoldRecord("https://legacy.hold.io", "alice", true)
214
-
recordJSON, _ := json.Marshal(holdRecord)
215
-
w.Header().Set("Content-Type", "application/json")
216
-
json.NewEncoder(w).Encode(map[string]any{
217
-
"records": []any{
218
-
map[string]any{
219
-
"uri": "at://did:plc:test123/io.atcr.hold/abc123",
220
-
"value": json.RawMessage(recordJSON),
221
-
},
222
-
},
223
-
})
224
-
return
225
-
}
226
-
w.WriteHeader(http.StatusNotFound)
227
-
}))
228
-
defer mockPDS.Close()
229
-
230
-
resolver := &NamespaceResolver{
231
-
defaultHoldDID: "did:web:default.atcr.io",
232
-
}
233
-
234
-
ctx := context.Background()
235
-
holdDID := resolver.findHoldDID(ctx, "did:plc:test123", mockPDS.URL)
236
-
237
-
// Legacy URL should be converted to DID
238
-
assert.Equal(t, "did:web:legacy.hold.io", holdDID, "should use legacy hold record and convert to DID")
239
-
}
240
-
241
202
// TestFindHoldDID_Priority tests the priority order
242
203
func TestFindHoldDID_Priority(t *testing.T) {
243
204
// Start a mock PDS server that returns both profile and hold records
···
248
209
w.Header().Set("Content-Type", "application/json")
249
210
json.NewEncoder(w).Encode(map[string]any{
250
211
"value": profile,
251
-
})
252
-
return
253
-
}
254
-
if r.URL.Path == "/xrpc/com.atproto.repo.listRecords" {
255
-
// Return hold record (should be ignored since profile exists)
256
-
holdRecord := atproto.NewHoldRecord("https://legacy.hold.io", "alice", true)
257
-
recordJSON, _ := json.Marshal(holdRecord)
258
-
w.Header().Set("Content-Type", "application/json")
259
-
json.NewEncoder(w).Encode(map[string]any{
260
-
"records": []any{
261
-
map[string]any{
262
-
"uri": "at://did:plc:test123/io.atcr.hold/abc123",
263
-
"value": json.RawMessage(recordJSON),
264
-
},
265
-
},
266
212
})
267
213
return
268
214
}
+5
-3
pkg/appview/storage/context.go
+5
-3
pkg/appview/storage/context.go
···
1
1
package storage
2
2
3
3
import (
4
+
"atcr.io/pkg/appview/readme"
4
5
"atcr.io/pkg/atproto"
5
6
"atcr.io/pkg/auth"
6
7
"atcr.io/pkg/auth/oauth"
···
27
28
AuthMethod string // Auth method used ("oauth" or "app_password")
28
29
29
30
// Shared services (same for all requests)
30
-
Database DatabaseMetrics // Metrics tracking database
31
-
Authorizer auth.HoldAuthorizer // Hold access authorization
32
-
Refresher *oauth.Refresher // OAuth session manager
31
+
Database DatabaseMetrics // Metrics tracking database
32
+
Authorizer auth.HoldAuthorizer // Hold access authorization
33
+
Refresher *oauth.Refresher // OAuth session manager
34
+
ReadmeFetcher *readme.Fetcher // README fetcher for repo pages
33
35
}
+218
-12
pkg/appview/storage/manifest_store.go
+218
-12
pkg/appview/storage/manifest_store.go
···
12
12
"net/http"
13
13
"strings"
14
14
"sync"
15
+
"time"
15
16
17
+
"atcr.io/pkg/appview/readme"
16
18
"atcr.io/pkg/atproto"
17
19
"github.com/distribution/distribution/v3"
18
20
"github.com/opencontainers/go-digest"
···
432
434
return
433
435
}
434
436
435
-
// Check for relevant annotations that we can use for repo page
436
-
description := manifestRecord.Annotations["org.opencontainers.image.description"]
437
-
if description == "" {
438
-
// No description annotation - nothing to create
439
-
return
440
-
}
441
-
442
437
// Check if repo page already exists (don't overwrite user's custom content)
443
438
rkey := s.ctx.Repository
444
439
_, err := s.ctx.ATProtoClient.GetRecord(ctx, atproto.RepoPageCollection, rkey)
···
454
449
return
455
450
}
456
451
457
-
// Create new repo page record from manifest annotations
458
-
// Note: Avatar is not extracted from annotations here - that's handled separately
459
-
// (would require uploading a blob if annotation contains a URL)
460
-
repoPage := atproto.NewRepoPageRecord(s.ctx.Repository, description, nil)
452
+
// Try to fetch README content from external sources
453
+
// Priority: io.atcr.readme annotation > derived from org.opencontainers.image.source > org.opencontainers.image.description
454
+
description := s.fetchReadmeContent(ctx, manifestRecord.Annotations)
455
+
456
+
// If no README content could be fetched, fall back to description annotation
457
+
if description == "" {
458
+
description = manifestRecord.Annotations["org.opencontainers.image.description"]
459
+
}
460
+
461
+
// Try to fetch and upload icon from io.atcr.icon annotation
462
+
var avatarRef *atproto.ATProtoBlobRef
463
+
if iconURL := manifestRecord.Annotations["io.atcr.icon"]; iconURL != "" {
464
+
avatarRef = s.fetchAndUploadIcon(ctx, iconURL)
465
+
}
466
+
467
+
// If no description and no icon, nothing to create
468
+
if description == "" && avatarRef == nil {
469
+
slog.Debug("No README, description, or icon found for repo page", "did", s.ctx.DID, "repository", s.ctx.Repository)
470
+
return
471
+
}
472
+
473
+
// Create new repo page record with description and optional avatar
474
+
repoPage := atproto.NewRepoPageRecord(s.ctx.Repository, description, avatarRef)
461
475
462
-
slog.Info("Creating repo page from manifest annotations", "did", s.ctx.DID, "repository", s.ctx.Repository)
476
+
slog.Info("Creating repo page from manifest annotations", "did", s.ctx.DID, "repository", s.ctx.Repository, "descriptionLength", len(description), "hasAvatar", avatarRef != nil)
463
477
464
478
_, err = s.ctx.ATProtoClient.PutRecord(ctx, atproto.RepoPageCollection, rkey, repoPage)
465
479
if err != nil {
···
469
483
470
484
slog.Info("Repo page created successfully", "did", s.ctx.DID, "repository", s.ctx.Repository)
471
485
}
486
+
487
+
// fetchReadmeContent attempts to fetch README content from external sources
488
+
// Priority: io.atcr.readme annotation > derived from org.opencontainers.image.source
489
+
// Returns the raw markdown content, or empty string if not available
490
+
func (s *ManifestStore) fetchReadmeContent(ctx context.Context, annotations map[string]string) string {
491
+
if s.ctx.ReadmeFetcher == nil {
492
+
return ""
493
+
}
494
+
495
+
// Create a context with timeout for README fetching (don't block push too long)
496
+
fetchCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
497
+
defer cancel()
498
+
499
+
// Priority 1: Direct README URL from io.atcr.readme annotation
500
+
if readmeURL := annotations["io.atcr.readme"]; readmeURL != "" {
501
+
content, err := s.fetchRawReadme(fetchCtx, readmeURL)
502
+
if err != nil {
503
+
slog.Debug("Failed to fetch README from io.atcr.readme annotation", "url", readmeURL, "error", err)
504
+
} else if content != "" {
505
+
slog.Info("Fetched README from io.atcr.readme annotation", "url", readmeURL, "length", len(content))
506
+
return content
507
+
}
508
+
}
509
+
510
+
// Priority 2: Derive README URL from org.opencontainers.image.source
511
+
if sourceURL := annotations["org.opencontainers.image.source"]; sourceURL != "" {
512
+
// Try main branch first, then master
513
+
for _, branch := range []string{"main", "master"} {
514
+
readmeURL := readme.DeriveReadmeURL(sourceURL, branch)
515
+
if readmeURL == "" {
516
+
continue
517
+
}
518
+
519
+
content, err := s.fetchRawReadme(fetchCtx, readmeURL)
520
+
if err != nil {
521
+
// Only log non-404 errors (404 is expected when trying main vs master)
522
+
if !readme.Is404(err) {
523
+
slog.Debug("Failed to fetch README from source URL", "url", readmeURL, "branch", branch, "error", err)
524
+
}
525
+
continue
526
+
}
527
+
528
+
if content != "" {
529
+
slog.Info("Fetched README from source URL", "sourceURL", sourceURL, "branch", branch, "length", len(content))
530
+
return content
531
+
}
532
+
}
533
+
}
534
+
535
+
return ""
536
+
}
537
+
538
+
// fetchRawReadme fetches raw markdown content from a URL
539
+
// Returns the raw markdown (not rendered HTML) for storage in the repo page record
540
+
func (s *ManifestStore) fetchRawReadme(ctx context.Context, readmeURL string) (string, error) {
541
+
// Use a simple HTTP client to fetch raw content
542
+
// We want raw markdown, not rendered HTML (the Fetcher renders to HTML)
543
+
req, err := http.NewRequestWithContext(ctx, "GET", readmeURL, nil)
544
+
if err != nil {
545
+
return "", fmt.Errorf("failed to create request: %w", err)
546
+
}
547
+
548
+
req.Header.Set("User-Agent", "ATCR-README-Fetcher/1.0")
549
+
550
+
client := &http.Client{
551
+
Timeout: 10 * time.Second,
552
+
CheckRedirect: func(req *http.Request, via []*http.Request) error {
553
+
if len(via) >= 5 {
554
+
return fmt.Errorf("too many redirects")
555
+
}
556
+
return nil
557
+
},
558
+
}
559
+
560
+
resp, err := client.Do(req)
561
+
if err != nil {
562
+
return "", fmt.Errorf("failed to fetch URL: %w", err)
563
+
}
564
+
defer resp.Body.Close()
565
+
566
+
if resp.StatusCode != http.StatusOK {
567
+
return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
568
+
}
569
+
570
+
// Limit content size to 100KB (repo page description has 100KB limit in lexicon)
571
+
limitedReader := io.LimitReader(resp.Body, 100*1024)
572
+
content, err := io.ReadAll(limitedReader)
573
+
if err != nil {
574
+
return "", fmt.Errorf("failed to read response body: %w", err)
575
+
}
576
+
577
+
return string(content), nil
578
+
}
579
+
580
+
// fetchAndUploadIcon fetches an image from a URL and uploads it as a blob to the user's PDS
581
+
// Returns the blob reference for use in the repo page record, or nil on error
582
+
func (s *ManifestStore) fetchAndUploadIcon(ctx context.Context, iconURL string) *atproto.ATProtoBlobRef {
583
+
// Create a context with timeout for icon fetching
584
+
fetchCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
585
+
defer cancel()
586
+
587
+
// Fetch the icon
588
+
req, err := http.NewRequestWithContext(fetchCtx, "GET", iconURL, nil)
589
+
if err != nil {
590
+
slog.Debug("Failed to create icon request", "url", iconURL, "error", err)
591
+
return nil
592
+
}
593
+
594
+
req.Header.Set("User-Agent", "ATCR-Icon-Fetcher/1.0")
595
+
596
+
client := &http.Client{
597
+
Timeout: 10 * time.Second,
598
+
CheckRedirect: func(req *http.Request, via []*http.Request) error {
599
+
if len(via) >= 5 {
600
+
return fmt.Errorf("too many redirects")
601
+
}
602
+
return nil
603
+
},
604
+
}
605
+
606
+
resp, err := client.Do(req)
607
+
if err != nil {
608
+
slog.Debug("Failed to fetch icon", "url", iconURL, "error", err)
609
+
return nil
610
+
}
611
+
defer resp.Body.Close()
612
+
613
+
if resp.StatusCode != http.StatusOK {
614
+
slog.Debug("Icon fetch returned non-OK status", "url", iconURL, "status", resp.StatusCode)
615
+
return nil
616
+
}
617
+
618
+
// Validate content type - only allow images
619
+
contentType := resp.Header.Get("Content-Type")
620
+
mimeType := detectImageMimeType(contentType, iconURL)
621
+
if mimeType == "" {
622
+
slog.Debug("Icon has unsupported content type", "url", iconURL, "contentType", contentType)
623
+
return nil
624
+
}
625
+
626
+
// Limit icon size to 3MB (matching lexicon maxSize)
627
+
limitedReader := io.LimitReader(resp.Body, 3*1024*1024)
628
+
iconData, err := io.ReadAll(limitedReader)
629
+
if err != nil {
630
+
slog.Debug("Failed to read icon data", "url", iconURL, "error", err)
631
+
return nil
632
+
}
633
+
634
+
if len(iconData) == 0 {
635
+
slog.Debug("Icon data is empty", "url", iconURL)
636
+
return nil
637
+
}
638
+
639
+
// Upload the icon as a blob to the user's PDS
640
+
blobRef, err := s.ctx.ATProtoClient.UploadBlob(ctx, iconData, mimeType)
641
+
if err != nil {
642
+
slog.Warn("Failed to upload icon blob", "url", iconURL, "error", err)
643
+
return nil
644
+
}
645
+
646
+
slog.Info("Uploaded icon blob", "url", iconURL, "size", len(iconData), "mimeType", mimeType, "cid", blobRef.Ref.Link)
647
+
return blobRef
648
+
}
649
+
650
+
// detectImageMimeType determines the MIME type for an image
651
+
// Uses Content-Type header first, then falls back to extension-based detection
652
+
// Only allows types accepted by the lexicon: image/png, image/jpeg, image/webp
653
+
func detectImageMimeType(contentType, url string) string {
654
+
// Check Content-Type header first
655
+
switch {
656
+
case strings.HasPrefix(contentType, "image/png"):
657
+
return "image/png"
658
+
case strings.HasPrefix(contentType, "image/jpeg"):
659
+
return "image/jpeg"
660
+
case strings.HasPrefix(contentType, "image/webp"):
661
+
return "image/webp"
662
+
}
663
+
664
+
// Fall back to URL extension detection
665
+
lowerURL := strings.ToLower(url)
666
+
switch {
667
+
case strings.HasSuffix(lowerURL, ".png"):
668
+
return "image/png"
669
+
case strings.HasSuffix(lowerURL, ".jpg"), strings.HasSuffix(lowerURL, ".jpeg"):
670
+
return "image/jpeg"
671
+
case strings.HasSuffix(lowerURL, ".webp"):
672
+
return "image/webp"
673
+
}
674
+
675
+
// Unknown or unsupported type - reject
676
+
return ""
677
+
}
-13
pkg/atproto/lexicon.go
-13
pkg/atproto/lexicon.go
···
18
18
// TagCollection is the collection name for image tags
19
19
TagCollection = "io.atcr.tag"
20
20
21
-
// HoldCollection is the collection name for storage holds (BYOS)
22
-
HoldCollection = "io.atcr.hold"
23
-
24
21
// HoldCrewCollection is the collection name for hold crew (membership) - LEGACY BYOS model
25
22
// Stored in owner's PDS for BYOS holds
26
23
HoldCrewCollection = "io.atcr.hold.crew"
···
313
310
CreatedAt time.Time `json:"createdAt"`
314
311
}
315
312
316
-
// NewHoldRecord creates a new hold record
317
-
func NewHoldRecord(endpoint, owner string, public bool) *HoldRecord {
318
-
return &HoldRecord{
319
-
Type: HoldCollection,
320
-
Endpoint: endpoint,
321
-
Owner: owner,
322
-
Public: public,
323
-
CreatedAt: time.Now(),
324
-
}
325
-
}
326
313
327
314
// SailorProfileRecord represents a user's profile with registry preferences
328
315
// Stored in the user's PDS to configure default hold and other settings
-50
pkg/atproto/lexicon_test.go
-50
pkg/atproto/lexicon_test.go
···
452
452
}
453
453
}
454
454
455
-
func TestNewHoldRecord(t *testing.T) {
456
-
tests := []struct {
457
-
name string
458
-
endpoint string
459
-
owner string
460
-
public bool
461
-
}{
462
-
{
463
-
name: "public hold",
464
-
endpoint: "https://hold1.example.com",
465
-
owner: "did:plc:alice123",
466
-
public: true,
467
-
},
468
-
{
469
-
name: "private hold",
470
-
endpoint: "https://hold2.example.com",
471
-
owner: "did:plc:bob456",
472
-
public: false,
473
-
},
474
-
}
475
-
476
-
for _, tt := range tests {
477
-
t.Run(tt.name, func(t *testing.T) {
478
-
before := time.Now()
479
-
record := NewHoldRecord(tt.endpoint, tt.owner, tt.public)
480
-
after := time.Now()
481
-
482
-
if record.Type != HoldCollection {
483
-
t.Errorf("Type = %v, want %v", record.Type, HoldCollection)
484
-
}
485
-
486
-
if record.Endpoint != tt.endpoint {
487
-
t.Errorf("Endpoint = %v, want %v", record.Endpoint, tt.endpoint)
488
-
}
489
-
490
-
if record.Owner != tt.owner {
491
-
t.Errorf("Owner = %v, want %v", record.Owner, tt.owner)
492
-
}
493
-
494
-
if record.Public != tt.public {
495
-
t.Errorf("Public = %v, want %v", record.Public, tt.public)
496
-
}
497
-
498
-
if record.CreatedAt.Before(before) || record.CreatedAt.After(after) {
499
-
t.Errorf("CreatedAt = %v, want between %v and %v", record.CreatedAt, before, after)
500
-
}
501
-
})
502
-
}
503
-
}
504
-
505
455
func TestNewSailorProfileRecord(t *testing.T) {
506
456
tests := []struct {
507
457
name string