···11+version: 1
22+name: remove_star_count_from_repository_stats
33+up: |
44+ -- Drop star_count column if it exists (SQLite 3.35.0+)
55+ ALTER TABLE repository_stats DROP COLUMN IF EXISTS star_count;
66+77+ -- Drop the old star_count index if it exists
88+ DROP INDEX IF EXISTS idx_repository_stats_star_count;
+77
pkg/appview/db/migrations/README.md
···11+# Database Migrations
22+33+This directory contains database migrations for the ATCR AppView database.
44+55+## Migration Format
66+77+Each migration is a YAML file with the following structure:
88+99+```yaml
1010+version: 1
1111+name: descriptive_migration_name
1212+up: |
1313+ SQL commands to apply the migration
1414+```
1515+1616+## Naming Convention
1717+1818+Migration files should be named: `{version:04d}_{name}.yaml`
1919+2020+Examples:
2121+- `0001_remove_star_count_from_repository_stats.yaml`
2222+- `0002_add_repository_labels.yaml`
2323+- `0003_create_webhooks_table.yaml`
2424+2525+## Creating a New Migration
2626+2727+1. **Choose the next version number** - Look at existing migrations and increment by 1
2828+2. **Create a new YAML file** with the naming convention above
2929+3. **Write your SQL** - Use the `|` block scalar for clean multi-line SQL
3030+4. **Use `IF EXISTS` / `IF NOT EXISTS`** where possible for idempotency
3131+3232+## Examples
3333+3434+### Simple single-statement migration:
3535+3636+```yaml
3737+version: 2
3838+name: add_repository_description_index
3939+up: |
4040+ CREATE INDEX IF NOT EXISTS idx_manifests_description ON manifests(description);
4141+```
4242+4343+### Complex multi-statement migration:
4444+4545+```yaml
4646+version: 3
4747+name: create_webhooks_table
4848+up: |
4949+ -- Create webhooks table
5050+ CREATE TABLE IF NOT EXISTS webhooks (
5151+ id INTEGER PRIMARY KEY AUTOINCREMENT,
5252+ url TEXT NOT NULL,
5353+ events TEXT NOT NULL,
5454+ created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
5555+ );
5656+5757+ -- Create index on URL for faster lookups
5858+ CREATE INDEX IF NOT EXISTS idx_webhooks_url ON webhooks(url);
5959+6060+ -- Create index on events for filtering
6161+ CREATE INDEX IF NOT EXISTS idx_webhooks_events ON webhooks(events);
6262+```
6363+6464+## How Migrations Run
6565+6666+1. Migrations are loaded from this directory on startup
6767+2. Sorted by version number (ascending)
6868+3. Each migration is checked against the `schema_migrations` table
6969+4. Only unapplied migrations are executed
7070+5. After successful execution, the version is recorded in `schema_migrations`
7171+7272+## Important Notes
7373+7474+- **Never modify existing migrations** - Once applied, they're immutable
7575+- **Test migrations** before committing - Ensure they work on existing databases
7676+- **Version numbers must be unique** - The migration system will fail if duplicates exist
7777+- **Migrations are run automatically** on `InitDB()` - No manual intervention needed
+1-1
pkg/appview/db/models.go
···8181type RepositoryStats struct {
8282 DID string `json:"did"`
8383 Repository string `json:"repository"`
8484- StarCount int `json:"star_count"`
8484+ StarCount int `json:"star_count"` // Calculated from stars table, not stored
8585 PullCount int `json:"pull_count"`
8686 LastPull *time.Time `json:"last_pull,omitempty"`
8787 PushCount int `json:"push_count"`
+109-4
pkg/appview/db/queries.go
···767767 var stats RepositoryStats
768768 var lastPullStr, lastPushStr sql.NullString
769769770770+ // Get pull/push stats from repository_stats, and star count from stars table
770771 err := db.QueryRow(`
771771- SELECT did, repository, star_count, pull_count, last_pull, push_count, last_push
772772- FROM repository_stats
773773- WHERE did = ? AND repository = ?
774774- `, did, repository).Scan(&stats.DID, &stats.Repository, &stats.StarCount, &stats.PullCount, &lastPullStr, &stats.PushCount, &lastPushStr)
772772+ SELECT
773773+ COALESCE(rs.did, ?) as did,
774774+ COALESCE(rs.repository, ?) as repository,
775775+ (SELECT COUNT(*) FROM stars WHERE owner_did = ? AND repository = ?) as star_count,
776776+ COALESCE(rs.pull_count, 0) as pull_count,
777777+ rs.last_pull,
778778+ COALESCE(rs.push_count, 0) as push_count,
779779+ rs.last_push
780780+ FROM (SELECT ? as did, ? as repository) AS placeholder
781781+ LEFT JOIN repository_stats rs ON rs.did = ? AND rs.repository = ?
782782+ `, did, repository, did, repository, did, repository, did, repository).Scan(&stats.DID, &stats.Repository, &stats.StarCount, &stats.PullCount, &lastPullStr, &stats.PushCount, &lastPushStr)
775783776784 if err == sql.ErrNoRows {
777785 // Return zero stats if no record exists yet
···838846 WHERE did = ? AND repository = ?
839847 `, did, repository)
840848 return err
849849+}
850850+851851+// UpsertStar inserts or updates a star record (idempotent)
852852+func UpsertStar(db *sql.DB, starrerDID, ownerDID, repository string, createdAt time.Time) error {
853853+ _, err := db.Exec(`
854854+ INSERT INTO stars (starrer_did, owner_did, repository, created_at)
855855+ VALUES (?, ?, ?, ?)
856856+ ON CONFLICT(starrer_did, owner_did, repository) DO UPDATE SET
857857+ created_at = excluded.created_at
858858+ `, starrerDID, ownerDID, repository, createdAt)
859859+ return err
860860+}
861861+862862+// DeleteStar deletes a star record
863863+func DeleteStar(db *sql.DB, starrerDID, ownerDID, repository string) error {
864864+ _, err := db.Exec(`
865865+ DELETE FROM stars
866866+ WHERE starrer_did = ? AND owner_did = ? AND repository = ?
867867+ `, starrerDID, ownerDID, repository)
868868+ return err
869869+}
870870+871871+// RebuildStarCount rebuilds the star count for a specific repository from the stars table
872872+func RebuildStarCount(db *sql.DB, ownerDID, repository string) error {
873873+ _, err := db.Exec(`
874874+ INSERT INTO repository_stats (did, repository, star_count)
875875+ VALUES (?, ?, (
876876+ SELECT COUNT(*) FROM stars
877877+ WHERE owner_did = ? AND repository = ?
878878+ ))
879879+ ON CONFLICT(did, repository) DO UPDATE SET
880880+ star_count = (
881881+ SELECT COUNT(*) FROM stars
882882+ WHERE owner_did = ? AND repository = ?
883883+ )
884884+ `, ownerDID, repository, ownerDID, repository, ownerDID, repository)
885885+ return err
886886+}
887887+888888+// GetStarsForDID returns all stars created by a specific DID (for backfill reconciliation)
889889+// Returns a map of (ownerDID, repository) -> createdAt
890890+func GetStarsForDID(db *sql.DB, starrerDID string) (map[string]time.Time, error) {
891891+ rows, err := db.Query(`
892892+ SELECT owner_did, repository, created_at
893893+ FROM stars
894894+ WHERE starrer_did = ?
895895+ `, starrerDID)
896896+ if err != nil {
897897+ return nil, err
898898+ }
899899+ defer rows.Close()
900900+901901+ stars := make(map[string]time.Time)
902902+ for rows.Next() {
903903+ var ownerDID, repository string
904904+ var createdAt time.Time
905905+ if err := rows.Scan(&ownerDID, &repository, &createdAt); err != nil {
906906+ return nil, err
907907+ }
908908+ key := fmt.Sprintf("%s/%s", ownerDID, repository)
909909+ stars[key] = createdAt
910910+ }
911911+912912+ return stars, rows.Err()
913913+}
914914+915915+// DeleteStarsNotInList deletes stars from the database that are not in the provided list
916916+// This is used during backfill reconciliation to remove stars that no longer exist on PDS
917917+func DeleteStarsNotInList(db *sql.DB, starrerDID string, foundStars map[string]time.Time) error {
918918+ // Get current stars in DB
919919+ currentStars, err := GetStarsForDID(db, starrerDID)
920920+ if err != nil {
921921+ return fmt.Errorf("failed to get current stars: %w", err)
922922+ }
923923+924924+ // Find stars to delete (in DB but not on PDS)
925925+ var toDelete []struct{ ownerDID, repository string }
926926+ for key := range currentStars {
927927+ if _, exists := foundStars[key]; !exists {
928928+ parts := strings.SplitN(key, "/", 2)
929929+ if len(parts) == 2 {
930930+ toDelete = append(toDelete, struct{ ownerDID, repository string }{
931931+ ownerDID: parts[0],
932932+ repository: parts[1],
933933+ })
934934+ }
935935+ }
936936+ }
937937+938938+ // Delete orphaned stars
939939+ for _, star := range toDelete {
940940+ if err := DeleteStar(db, starrerDID, star.ownerDID, star.repository); err != nil {
941941+ return fmt.Errorf("failed to delete star: %w", err)
942942+ }
943943+ }
944944+945945+ return nil
841946}
842947843948// IncrementPullCount increments the pull count for a repository
+122-2
pkg/appview/db/schema.go
···2233import (
44 "database/sql"
55+ "fmt"
66+ "os"
77+ "path/filepath"
88+ "sort"
59610 _ "github.com/mattn/go-sqlite3"
1111+ "go.yaml.in/yaml/v4"
712)
813914const schema = `
1515+CREATE TABLE IF NOT EXISTS schema_migrations (
1616+ version INTEGER PRIMARY KEY,
1717+ applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
1818+);
1919+1020CREATE TABLE IF NOT EXISTS users (
1121 did TEXT PRIMARY KEY,
1222 handle TEXT NOT NULL,
···144154CREATE TABLE IF NOT EXISTS repository_stats (
145155 did TEXT NOT NULL,
146156 repository TEXT NOT NULL,
147147- star_count INTEGER NOT NULL DEFAULT 0,
148157 pull_count INTEGER NOT NULL DEFAULT 0,
149158 last_pull TIMESTAMP,
150159 push_count INTEGER NOT NULL DEFAULT 0,
···153162 FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE
154163);
155164CREATE INDEX IF NOT EXISTS idx_repository_stats_did ON repository_stats(did);
156156-CREATE INDEX IF NOT EXISTS idx_repository_stats_star_count ON repository_stats(star_count DESC);
157165CREATE INDEX IF NOT EXISTS idx_repository_stats_pull_count ON repository_stats(pull_count DESC);
166166+167167+CREATE TABLE IF NOT EXISTS stars (
168168+ starrer_did TEXT NOT NULL,
169169+ owner_did TEXT NOT NULL,
170170+ repository TEXT NOT NULL,
171171+ created_at TIMESTAMP NOT NULL,
172172+ PRIMARY KEY(starrer_did, owner_did, repository),
173173+ FOREIGN KEY(starrer_did) REFERENCES users(did) ON DELETE CASCADE,
174174+ FOREIGN KEY(owner_did) REFERENCES users(did) ON DELETE CASCADE
175175+);
176176+CREATE INDEX IF NOT EXISTS idx_stars_owner_repo ON stars(owner_did, repository);
177177+CREATE INDEX IF NOT EXISTS idx_stars_starrer ON stars(starrer_did);
158178`
159179160180// InitDB initializes the SQLite database with the schema
···174194 return nil, err
175195 }
176196197197+ // Run migrations
198198+ if err := runMigrations(db); err != nil {
199199+ return nil, err
200200+ }
201201+177202 return db, nil
203203+}
204204+205205+// Migration represents a database migration
206206+type Migration struct {
207207+ Version int `yaml:"version"`
208208+ Name string `yaml:"name"`
209209+ Up string `yaml:"up"`
210210+}
211211+212212+// runMigrations applies any pending database migrations
213213+func runMigrations(db *sql.DB) error {
214214+ // Load migrations from files
215215+ migrations, err := loadMigrations()
216216+ if err != nil {
217217+ return fmt.Errorf("failed to load migrations: %w", err)
218218+ }
219219+220220+ // Sort migrations by version
221221+ sort.Slice(migrations, func(i, j int) bool {
222222+ return migrations[i].Version < migrations[j].Version
223223+ })
224224+225225+ for _, m := range migrations {
226226+ // Check if migration already applied
227227+ var count int
228228+ err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations WHERE version = ?", m.Version).Scan(&count)
229229+ if err != nil {
230230+ return fmt.Errorf("failed to check migration status: %w", err)
231231+ }
232232+233233+ if count > 0 {
234234+ // Migration already applied
235235+ continue
236236+ }
237237+238238+ // Apply migration
239239+ fmt.Printf("Applying migration %d: %s\n", m.Version, m.Name)
240240+ if _, err := db.Exec(m.Up); err != nil {
241241+ return fmt.Errorf("failed to apply migration %d (%s): %w", m.Version, m.Name, err)
242242+ }
243243+244244+ // Record migration
245245+ if _, err := db.Exec("INSERT INTO schema_migrations (version) VALUES (?)", m.Version); err != nil {
246246+ return fmt.Errorf("failed to record migration %d: %w", m.Version, err)
247247+ }
248248+249249+ fmt.Printf("Migration %d applied successfully\n", m.Version)
250250+ }
251251+252252+ return nil
253253+}
254254+255255+// loadMigrations loads all migration files from the migrations directory
256256+func loadMigrations() ([]Migration, error) {
257257+ // Get the path to the migrations directory
258258+ // Try relative to working directory first, then relative to this file
259259+ migrationsDir := "pkg/appview/db/migrations"
260260+ if _, err := os.Stat(migrationsDir); os.IsNotExist(err) {
261261+ // Try embedded path (when running from different directory)
262262+ migrationsDir = filepath.Join(".", "migrations")
263263+ }
264264+265265+ // Read all .yaml files in the migrations directory
266266+ files, err := filepath.Glob(filepath.Join(migrationsDir, "*.yaml"))
267267+ if err != nil {
268268+ return nil, fmt.Errorf("failed to list migration files: %w", err)
269269+ }
270270+271271+ var migrations []Migration
272272+ for _, file := range files {
273273+ data, err := os.ReadFile(file)
274274+ if err != nil {
275275+ return nil, fmt.Errorf("failed to read migration file %s: %w", file, err)
276276+ }
277277+278278+ var m Migration
279279+ if err := yaml.Unmarshal(data, &m); err != nil {
280280+ return nil, fmt.Errorf("failed to parse migration file %s: %w", file, err)
281281+ }
282282+283283+ // Validate migration
284284+ if m.Version <= 0 {
285285+ return nil, fmt.Errorf("invalid migration version in %s: %d", file, m.Version)
286286+ }
287287+ if m.Name == "" {
288288+ return nil, fmt.Errorf("missing migration name in %s", file)
289289+ }
290290+ if m.Up == "" {
291291+ return nil, fmt.Errorf("missing migration 'up' SQL in %s", file)
292292+ }
293293+294294+ migrations = append(migrations, m)
295295+ }
296296+297297+ return migrations, nil
178298}
+19-4
pkg/appview/jetstream/backfill.go
···145145 // Track which records exist on the PDS for reconciliation
146146 var foundManifestDigests []string
147147 var foundTags []struct{ Repository, Tag string }
148148+ foundStars := make(map[string]time.Time) // key: "ownerDID/repository", value: createdAt
148149149150 // Paginate through all records for this repo
150151 for {
···169170 Tag: tagRecord.Tag,
170171 })
171172 }
173173+ } else if collection == atproto.StarCollection {
174174+ var starRecord atproto.StarRecord
175175+ if err := json.Unmarshal(record.Value, &starRecord); err == nil {
176176+ key := fmt.Sprintf("%s/%s", starRecord.Subject.DID, starRecord.Subject.Repository)
177177+ foundStars[key] = starRecord.CreatedAt
178178+ }
172179 }
173180174181 if err := b.processRecord(ctx, did, collection, &record); err != nil {
···187194 }
188195189196 // Reconcile deletions - remove records from DB that no longer exist on PDS
190190- if err := b.reconcileDeletions(did, collection, foundManifestDigests, foundTags); err != nil {
197197+ if err := b.reconcileDeletions(did, collection, foundManifestDigests, foundTags, foundStars); err != nil {
191198 fmt.Printf("WARNING: Failed to reconcile deletions for %s: %v\n", did, err)
192199 }
193200···195202}
196203197204// reconcileDeletions removes records from the database that no longer exist on the PDS
198198-func (b *BackfillWorker) reconcileDeletions(did, collection string, foundManifestDigests []string, foundTags []struct{ Repository, Tag string }) error {
205205+func (b *BackfillWorker) reconcileDeletions(did, collection string, foundManifestDigests []string, foundTags []struct{ Repository, Tag string }, foundStars map[string]time.Time) error {
199206 switch collection {
200207 case atproto.ManifestCollection:
201208 // Get current manifests in DB
···231238 deleted := len(dbTags) - len(foundTags)
232239 if deleted > 0 {
233240 fmt.Printf("Backfill: Deleted %d orphaned tags for %s\n", deleted, did)
241241+ }
242242+243243+ case atproto.StarCollection:
244244+ // Reconcile stars - delete stars that no longer exist on PDS
245245+ // Star counts will be calculated on demand from the stars table
246246+ if err := db.DeleteStarsNotInList(b.db, did, foundStars); err != nil {
247247+ return fmt.Errorf("failed to delete orphaned stars: %w", err)
234248 }
235249 }
236250···336350 return fmt.Errorf("failed to unmarshal star: %w", err)
337351 }
338352339339- // Increment star count for the repository being starred
353353+ // Upsert the star record (idempotent - won't duplicate)
340354 // The DID here is the starrer (user who starred)
341355 // The subject contains the owner DID and repository
342342- return db.IncrementStarCount(b.db, starRecord.Subject.DID, starRecord.Subject.Repository)
356356+ // Star count will be calculated on demand from the stars table
357357+ return db.UpsertStar(b.db, did, starRecord.Subject.DID, starRecord.Subject.Repository, starRecord.CreatedAt)
343358}
344359345360// ensureUser resolves and upserts a user by DID
+10-17
pkg/appview/jetstream/worker.go
···415415 }
416416417417 if commit.Operation == "delete" {
418418- // Unstar - parse the record to get the subject (owner DID and repository)
419419- var starRecord atproto.StarRecord
420420- if commit.Record != nil {
421421- recordBytes, err := json.Marshal(commit.Record)
422422- if err != nil {
423423- return fmt.Errorf("failed to marshal record: %w", err)
424424- }
425425- if err := json.Unmarshal(recordBytes, &starRecord); err != nil {
426426- return fmt.Errorf("failed to unmarshal star: %w", err)
427427- }
418418+ // Unstar - parse the rkey to get the subject (owner DID and repository)
419419+ // Delete events don't include the full record, but the rkey contains the info we need
420420+ ownerDID, repository, err := atproto.ParseStarRecordKey(commit.RKey)
421421+ if err != nil {
422422+ return fmt.Errorf("failed to parse star rkey: %w", err)
423423+ }
428424429429- // Decrement star count
430430- return db.DecrementStarCount(w.db, starRecord.Subject.DID, starRecord.Subject.Repository)
431431- }
432432- // If no record data, we can't determine what was unstarred
433433- return nil
425425+ // Delete the star record
426426+ return db.DeleteStar(w.db, commit.DID, ownerDID, repository)
434427 }
435428436429 // Parse star record
···447440 return nil
448441 }
449442450450- // Increment star count for the repository being starred
451451- return db.IncrementStarCount(w.db, starRecord.Subject.DID, starRecord.Subject.Repository)
443443+ // Upsert the star record (idempotent - star count will be calculated on demand)
444444+ return db.UpsertStar(w.db, commit.DID, starRecord.Subject.DID, starRecord.Subject.Repository, starRecord.CreatedAt)
452445}
453446454447// JetstreamEvent represents a Jetstream event
+2-2
pkg/appview/static/js/app.js
···172172 starCountEl.textContent = Math.max(0, currentCount - 1);
173173 }
174174175175- // Refresh actual count from server (will correct if optimistic update was wrong)
176176- await loadStarCount(handle, repository);
175175+ // Don't fetch count immediately - trust the optimistic update
176176+ // The actual count will be correct on next page load
177177178178 } catch (err) {
179179 console.error('Error toggling star:', err);
+17
pkg/atproto/lexicon.go
···33import (
44 "encoding/base64"
55 "encoding/json"
66+ "fmt"
77+ "strings"
68 "time"
79)
810···302304 combined := ownerDID + "/" + repository
303305 return base64.RawURLEncoding.EncodeToString([]byte(combined))
304306}
307307+308308+// ParseStarRecordKey decodes a star record key back to ownerDID and repository
309309+func ParseStarRecordKey(rkey string) (ownerDID, repository string, err error) {
310310+ decoded, err := base64.RawURLEncoding.DecodeString(rkey)
311311+ if err != nil {
312312+ return "", "", fmt.Errorf("failed to decode star rkey: %w", err)
313313+ }
314314+315315+ parts := strings.SplitN(string(decoded), "/", 2)
316316+ if len(parts) != 2 {
317317+ return "", "", fmt.Errorf("invalid star rkey format: %s", string(decoded))
318318+ }
319319+320320+ return parts[0], parts[1], nil
321321+}