experiments in a post-browser web
10
fork

Configure Feed

Select the types of activity you want to include in your feed.

WIP: URL/History unification migration - all phases implemented

+952 -47
+31 -28
app/cmd/commands/history.js
··· 1 1 /** 2 - * History command - search and open pages from address history 3 - * Addresses are sorted by visitCount (frecency) 2 + * History command - search and open pages from URL history 3 + * Uses unified items table with frecency scoring 4 4 */ 5 5 import windows from '../../windows.js'; 6 6 import api from '../../api.js'; 7 7 8 8 /** 9 - * Get addresses sorted by visit count (frecency) 9 + * Get URL items sorted by frecency score 10 10 * Optionally filter by search term 11 11 */ 12 12 const getHistory = async (searchTerm = '', limit = 20) => { 13 - const result = await api.datastore.queryAddresses({}); 14 - if (!result.success) return []; 15 - 16 - let addresses = result.data; 13 + // Use the new unified items API with frecency sorting 14 + const filter = { 15 + type: 'url', 16 + sortBy: 'frecency', 17 + limit, 18 + }; 17 19 18 - // Filter by search term if provided 20 + // Add search filter if provided 19 21 if (searchTerm) { 20 - const lower = searchTerm.toLowerCase(); 21 - addresses = addresses.filter(addr => { 22 - const uri = (addr.uri || '').toLowerCase(); 23 - const title = (addr.title || '').toLowerCase(); 24 - const domain = (addr.domain || '').toLowerCase(); 25 - return uri.includes(lower) || title.includes(lower) || domain.includes(lower); 26 - }); 22 + filter.search = searchTerm; 27 23 } 28 24 29 - // Sort by visitCount descending (frecency) 30 - addresses.sort((a, b) => (b.visitCount || 0) - (a.visitCount || 0)); 25 + const result = await api.datastore.queryItems(filter); 26 + if (!result.success) return []; 31 27 32 - return addresses.slice(0, limit); 28 + // Transform items to the shape expected by the rest of the code 29 + return result.data.map(item => ({ 30 + uri: item.content || '', 31 + title: item.title || '', 32 + domain: item.domain || '', 33 + visitCount: item.visitCount || 0, 34 + frecencyScore: item.frecencyScore || 0, 35 + })); 33 36 }; 34 37 35 38 /** 36 - * Open an address from history 39 + * Open a URL from history 37 40 */ 38 41 const openFromHistory = async (uri) => { 39 42 try { ··· 58 61 name: 'history', 59 62 async execute(ctx) { 60 63 if (ctx.search) { 61 - // Search provided - find matching address and open it 64 + // Search provided - find matching item and open it 62 65 const matches = await getHistory(ctx.search, 1); 63 66 if (matches.length > 0) { 64 67 await openFromHistory(matches[0].uri); ··· 69 72 // No search - just log recent history 70 73 const recent = await getHistory('', 10); 71 74 console.log('Recent history:'); 72 - recent.forEach((addr, i) => { 73 - console.log(`${i + 1}. [${addr.visitCount || 0}] ${addr.title || addr.uri}`); 75 + recent.forEach((item, i) => { 76 + console.log(`${i + 1}. [${item.frecencyScore}] ${item.title || item.uri}`); 74 77 }); 75 78 } 76 79 } ··· 86 89 const history = await getHistory('', 50); // Get more entries 87 90 console.log('Adding history entries as commands:', history.length); 88 91 89 - history.forEach(addr => { 92 + history.forEach(item => { 90 93 // Use the URI as the command name so it's searchable 91 94 addCommand({ 92 - name: addr.uri, 95 + name: item.uri, 93 96 async execute(ctx) { 94 - await openFromHistory(addr.uri); 97 + await openFromHistory(item.uri); 95 98 } 96 99 }); 97 100 98 101 // Also add title as a command if it exists and is different 99 - if (addr.title && addr.title !== addr.uri) { 102 + if (item.title && item.title !== item.uri) { 100 103 addCommand({ 101 - name: addr.title, 104 + name: item.title, 102 105 async execute(ctx) { 103 - await openFromHistory(addr.uri); 106 + await openFromHistory(item.uri); 104 107 } 105 108 }); 106 109 }
+730 -1
backend/electron/datastore.ts
··· 25 25 ItemType, 26 26 ItemOptions, 27 27 ItemFilter, 28 + ItemVisit, 29 + ItemVisitFilter, 30 + ItemVisitOptions, 28 31 } from '../types/index.js'; 29 32 import { tableNames } from '../types/index.js'; 30 33 import { DEBUG } from './config.js'; ··· 283 286 key TEXT PRIMARY KEY, 284 287 value TEXT 285 288 ); 289 + 290 + CREATE TABLE IF NOT EXISTS item_visits ( 291 + id TEXT PRIMARY KEY, 292 + itemId TEXT NOT NULL, 293 + timestamp INTEGER NOT NULL, 294 + duration INTEGER DEFAULT 0, 295 + source TEXT DEFAULT 'direct', 296 + sourceId TEXT DEFAULT '', 297 + windowType TEXT DEFAULT 'main', 298 + metadata TEXT DEFAULT '{}', 299 + scrollDepth INTEGER DEFAULT 0, 300 + interacted INTEGER DEFAULT 0, 301 + prevId TEXT DEFAULT NULL, 302 + nextId TEXT DEFAULT NULL, 303 + FOREIGN KEY(itemId) REFERENCES items(id) 304 + ); 305 + CREATE INDEX IF NOT EXISTS idx_item_visits_itemId ON item_visits(itemId); 306 + CREATE INDEX IF NOT EXISTS idx_item_visits_timestamp ON item_visits(timestamp); 307 + CREATE INDEX IF NOT EXISTS idx_item_visits_prevId ON item_visits(prevId); 308 + CREATE INDEX IF NOT EXISTS idx_item_visits_nextId ON item_visits(nextId); 309 + 310 + CREATE TABLE IF NOT EXISTS item_groups ( 311 + id TEXT PRIMARY KEY, 312 + name TEXT NOT NULL, 313 + description TEXT DEFAULT '', 314 + type TEXT DEFAULT 'manual', 315 + query TEXT DEFAULT '', 316 + metadata TEXT DEFAULT '{}', 317 + createdAt INTEGER NOT NULL, 318 + updatedAt INTEGER NOT NULL, 319 + deletedAt INTEGER DEFAULT 0 320 + ); 321 + CREATE INDEX IF NOT EXISTS idx_item_groups_type ON item_groups(type); 322 + CREATE INDEX IF NOT EXISTS idx_item_groups_deletedAt ON item_groups(deletedAt); 323 + 324 + CREATE TABLE IF NOT EXISTS item_group_members ( 325 + id TEXT PRIMARY KEY, 326 + groupId TEXT NOT NULL, 327 + itemId TEXT NOT NULL, 328 + position INTEGER DEFAULT 0, 329 + createdAt INTEGER NOT NULL, 330 + FOREIGN KEY(groupId) REFERENCES item_groups(id), 331 + FOREIGN KEY(itemId) REFERENCES items(id) 332 + ); 333 + CREATE INDEX IF NOT EXISTS idx_item_group_members_groupId ON item_group_members(groupId); 334 + CREATE INDEX IF NOT EXISTS idx_item_group_members_itemId ON item_group_members(itemId); 335 + CREATE UNIQUE INDEX IF NOT EXISTS idx_item_group_members_unique ON item_group_members(groupId, itemId); 286 336 `; 287 337 288 338 // Module state ··· 304 354 migrateAddressesToItems(); 305 355 migrateVisitChaining(); 306 356 migrateDeduplicateItems(); 357 + migrateItemFrecencyColumns(); 358 + migrateAllAddressesToItems(); 359 + migrateVisitsToItemVisits(); 307 360 308 361 // Check and write datastore version 309 362 checkAndWriteDatastoreVersion(); ··· 893 946 } 894 947 } 895 948 949 + /** 950 + * Add frecencyScore, title, domain, favicon columns to items table for URL history unification. 951 + * These columns are local-only (not synced) and support frecency-based sorting. 952 + */ 953 + function migrateItemFrecencyColumns(): void { 954 + if (!db) return; 955 + 956 + const columns = db.prepare(`PRAGMA table_info(items)`).all() as { name: string }[]; 957 + const hasFrecencyScore = columns.some(col => col.name === 'frecencyScore'); 958 + 959 + if (!hasFrecencyScore) { 960 + DEBUG && console.log('main', 'Adding frecency columns to items table'); 961 + try { 962 + db.exec(`ALTER TABLE items ADD COLUMN frecencyScore INTEGER DEFAULT 0`); 963 + db.exec(`ALTER TABLE items ADD COLUMN title TEXT DEFAULT ''`); 964 + db.exec(`ALTER TABLE items ADD COLUMN domain TEXT DEFAULT ''`); 965 + db.exec(`ALTER TABLE items ADD COLUMN favicon TEXT DEFAULT ''`); 966 + db.exec(`CREATE INDEX IF NOT EXISTS idx_items_frecencyScore ON items(frecencyScore DESC)`); 967 + db.exec(`CREATE INDEX IF NOT EXISTS idx_items_domain ON items(domain)`); 968 + } catch (error) { 969 + DEBUG && console.log('main', `Item frecency columns migration:`, (error as Error).message); 970 + } 971 + } 972 + 973 + // Always ensure indexes exist 974 + try { 975 + db.exec(`CREATE INDEX IF NOT EXISTS idx_items_frecencyScore ON items(frecencyScore DESC)`); 976 + db.exec(`CREATE INDEX IF NOT EXISTS idx_items_domain ON items(domain)`); 977 + } catch (error) { 978 + DEBUG && console.log('main', `Item frecency indexes:`, (error as Error).message); 979 + } 980 + } 981 + 982 + /** 983 + * Calculate frecency score for an item based on visit history. 984 + * Uses a time-decay algorithm where recent visits contribute more. 985 + */ 986 + export function calculateItemFrecency(visits: Array<{ timestamp: number; interacted: number; source: string }>): number { 987 + let score = 0; 988 + for (const visit of visits) { 989 + const ageDays = (Date.now() - visit.timestamp) / (1000 * 60 * 60 * 24); 990 + const decay = 1 / (1 + Math.pow(ageDays / 7, 0.5)); 991 + // Weight: interacted visits count more, direct navigations count less than link clicks 992 + const weight = visit.interacted ? 2 : (visit.source === 'direct' ? 0.5 : 1); 993 + score += weight * decay; 994 + } 995 + return Math.round(score * 10); 996 + } 997 + 998 + /** 999 + * Migrate ALL addresses to items (not just tagged ones). 1000 + * This extends the earlier migrateAddressesToItems() which only migrated tagged addresses. 1001 + * Creates items for all addresses and tracks address.id → item.id mapping for visit migration. 1002 + */ 1003 + function migrateAllAddressesToItems(): void { 1004 + if (!db) return; 1005 + 1006 + const MIGRATION_ID = 'all_addresses_to_items_v1'; 1007 + 1008 + // Check if already migrated 1009 + const migrationRecord = db.prepare('SELECT * FROM migrations WHERE id = ?').get(MIGRATION_ID) as { status: string } | undefined; 1010 + if (migrationRecord && migrationRecord.status === 'complete') { 1011 + DEBUG && console.log('main', 'All addresses to items migration already complete'); 1012 + return; 1013 + } 1014 + 1015 + // Get ALL addresses (not just tagged ones) 1016 + const allAddresses = db.prepare('SELECT * FROM addresses').all() as Address[]; 1017 + 1018 + if (allAddresses.length === 0) { 1019 + db.prepare('INSERT OR REPLACE INTO migrations (id, status, completedAt) VALUES (?, ?, ?)').run(MIGRATION_ID, 'complete', Date.now()); 1020 + DEBUG && console.log('main', 'No addresses to migrate'); 1021 + return; 1022 + } 1023 + 1024 + DEBUG && console.log('main', `Migrating ${allAddresses.length} addresses to items table`); 1025 + 1026 + let createdCount = 0; 1027 + let mergedCount = 0; 1028 + 1029 + // Build address.id → item.id mapping for visit migration 1030 + const addressToItemMap: Record<string, string> = {}; 1031 + 1032 + for (const addr of allAddresses) { 1033 + // Check if item with this URL already exists 1034 + const existingItem = db.prepare('SELECT * FROM items WHERE type = ? AND content = ? AND deletedAt = 0').get('url', addr.uri) as Item | undefined; 1035 + 1036 + if (existingItem) { 1037 + // Map to existing item, merge metadata 1038 + addressToItemMap[addr.id] = existingItem.id; 1039 + 1040 + // Merge metadata: combine address metadata with item metadata 1041 + let itemMeta: Record<string, unknown> = {}; 1042 + try { 1043 + itemMeta = typeof existingItem.metadata === 'string' ? JSON.parse(existingItem.metadata) : existingItem.metadata || {}; 1044 + } catch { /* ignore */ } 1045 + 1046 + // Update with address data if item is missing it 1047 + const updates: string[] = []; 1048 + const values: unknown[] = []; 1049 + 1050 + if (!itemMeta.title && addr.title) { 1051 + itemMeta.title = addr.title; 1052 + } 1053 + if (!itemMeta.favicon && addr.favicon) { 1054 + itemMeta.favicon = addr.favicon; 1055 + } 1056 + 1057 + // Also set the denormalized columns 1058 + if (addr.title) { 1059 + updates.push('title = ?'); 1060 + values.push(addr.title); 1061 + } 1062 + if (addr.domain) { 1063 + updates.push('domain = ?'); 1064 + values.push(addr.domain); 1065 + } 1066 + if (addr.favicon) { 1067 + updates.push('favicon = ?'); 1068 + values.push(addr.favicon); 1069 + } 1070 + 1071 + // Merge visit stats (take max) 1072 + if ((addr.visitCount || 0) > (existingItem.visitCount || 0)) { 1073 + updates.push('visitCount = ?'); 1074 + values.push(addr.visitCount); 1075 + } 1076 + if ((addr.lastVisitAt || 0) > (existingItem.lastVisitAt || 0)) { 1077 + updates.push('lastVisitAt = ?'); 1078 + values.push(addr.lastVisitAt); 1079 + } 1080 + if (addr.starred && !existingItem.starred) { 1081 + updates.push('starred = ?'); 1082 + values.push(1); 1083 + } 1084 + 1085 + if (updates.length > 0) { 1086 + updates.push('metadata = ?'); 1087 + values.push(JSON.stringify(itemMeta)); 1088 + updates.push('updatedAt = ?'); 1089 + values.push(now()); 1090 + values.push(existingItem.id); 1091 + 1092 + db.prepare(`UPDATE items SET ${updates.join(', ')} WHERE id = ?`).run(...values); 1093 + } 1094 + 1095 + mergedCount++; 1096 + } else { 1097 + // Create new item for this URL 1098 + const itemId = generateId('item'); 1099 + const timestamp = now(); 1100 + addressToItemMap[addr.id] = itemId; 1101 + 1102 + // Build metadata from address 1103 + const metadata: Record<string, unknown> = {}; 1104 + if (addr.title) metadata.title = addr.title; 1105 + if (addr.description) metadata.description = addr.description; 1106 + if (addr.favicon) metadata.favicon = addr.favicon; 1107 + if (addr.metadata) { 1108 + try { 1109 + const addrMeta = typeof addr.metadata === 'string' ? JSON.parse(addr.metadata) : addr.metadata; 1110 + Object.assign(metadata, addrMeta); 1111 + } catch { /* ignore invalid JSON */ } 1112 + } 1113 + 1114 + db.prepare(` 1115 + INSERT INTO items (id, type, content, mimeType, metadata, syncId, syncSource, createdAt, updatedAt, deletedAt, starred, archived, visitCount, lastVisitAt, frecencyScore, title, domain, favicon) 1116 + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, 0, ?, ?, ?) 1117 + `).run( 1118 + itemId, 1119 + 'url', 1120 + addr.uri, 1121 + addr.mimeType || 'text/html', 1122 + JSON.stringify(metadata), 1123 + '', 1124 + '', 1125 + addr.createdAt || timestamp, 1126 + addr.updatedAt || timestamp, 1127 + addr.starred || 0, 1128 + addr.archived || 0, 1129 + addr.visitCount || 0, 1130 + addr.lastVisitAt || 0, 1131 + addr.title || '', 1132 + addr.domain || '', 1133 + addr.favicon || '' 1134 + ); 1135 + 1136 + createdCount++; 1137 + 1138 + // Copy tag associations from address_tags to item_tags 1139 + const addressTags = db.prepare('SELECT * FROM address_tags WHERE addressId = ?').all(addr.id) as AddressTag[]; 1140 + for (const at of addressTags) { 1141 + const existingLink = db.prepare('SELECT * FROM item_tags WHERE itemId = ? AND tagId = ?').get(itemId, at.tagId); 1142 + if (!existingLink) { 1143 + const linkId = generateId('item_tag'); 1144 + db.prepare('INSERT INTO item_tags (id, itemId, tagId, createdAt) VALUES (?, ?, ?, ?)').run( 1145 + linkId, 1146 + itemId, 1147 + at.tagId, 1148 + at.createdAt || now() 1149 + ); 1150 + } 1151 + } 1152 + } 1153 + } 1154 + 1155 + // Store the mapping for visit migration 1156 + db.prepare('INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)').run('address_to_item_map', JSON.stringify(addressToItemMap)); 1157 + 1158 + // Mark migration as complete 1159 + db.prepare('INSERT OR REPLACE INTO migrations (id, status, completedAt) VALUES (?, ?, ?)').run(MIGRATION_ID, 'complete', Date.now()); 1160 + DEBUG && console.log('main', `Migrated addresses to items: ${createdCount} created, ${mergedCount} merged`); 1161 + } 1162 + 1163 + /** 1164 + * Migrate visits from visits table to item_visits table. 1165 + * Uses the address_to_item_map created by migrateAllAddressesToItems(). 1166 + * Preserves visit chaining (prevId/nextId). 1167 + */ 1168 + function migrateVisitsToItemVisits(): void { 1169 + if (!db) return; 1170 + 1171 + const MIGRATION_ID = 'visits_to_item_visits_v1'; 1172 + 1173 + // Check if already migrated 1174 + const migrationRecord = db.prepare('SELECT * FROM migrations WHERE id = ?').get(MIGRATION_ID) as { status: string } | undefined; 1175 + if (migrationRecord && migrationRecord.status === 'complete') { 1176 + DEBUG && console.log('main', 'Visits to item_visits migration already complete'); 1177 + return; 1178 + } 1179 + 1180 + // Load address → item mapping 1181 + const mapRow = db.prepare('SELECT value FROM settings WHERE key = ?').get('address_to_item_map') as { value: string } | undefined; 1182 + if (!mapRow) { 1183 + DEBUG && console.log('main', 'No address_to_item_map found, skipping visit migration'); 1184 + db.prepare('INSERT OR REPLACE INTO migrations (id, status, completedAt) VALUES (?, ?, ?)').run(MIGRATION_ID, 'complete', Date.now()); 1185 + return; 1186 + } 1187 + 1188 + const addressToItemMap: Record<string, string> = JSON.parse(mapRow.value); 1189 + 1190 + // Get all visits 1191 + const allVisits = db.prepare('SELECT * FROM visits ORDER BY timestamp ASC').all() as Visit[]; 1192 + 1193 + if (allVisits.length === 0) { 1194 + db.prepare('INSERT OR REPLACE INTO migrations (id, status, completedAt) VALUES (?, ?, ?)').run(MIGRATION_ID, 'complete', Date.now()); 1195 + DEBUG && console.log('main', 'No visits to migrate'); 1196 + return; 1197 + } 1198 + 1199 + DEBUG && console.log('main', `Migrating ${allVisits.length} visits to item_visits table`); 1200 + 1201 + // Build old visit.id → new item_visit.id mapping for chaining 1202 + const visitIdMap: Record<string, string> = {}; 1203 + let migratedCount = 0; 1204 + let skippedCount = 0; 1205 + 1206 + // First pass: create all item_visits without chaining 1207 + for (const visit of allVisits) { 1208 + const itemId = addressToItemMap[visit.addressId]; 1209 + if (!itemId) { 1210 + // Address wasn't migrated (shouldn't happen, but handle gracefully) 1211 + skippedCount++; 1212 + continue; 1213 + } 1214 + 1215 + // Check if this visit was already migrated 1216 + const existingItemVisit = db.prepare('SELECT id FROM item_visits WHERE timestamp = ? AND itemId = ?').get(visit.timestamp, itemId); 1217 + if (existingItemVisit) { 1218 + visitIdMap[visit.id] = (existingItemVisit as { id: string }).id; 1219 + continue; 1220 + } 1221 + 1222 + const itemVisitId = generateId('item_visit'); 1223 + visitIdMap[visit.id] = itemVisitId; 1224 + 1225 + db.prepare(` 1226 + INSERT INTO item_visits (id, itemId, timestamp, duration, source, sourceId, windowType, metadata, scrollDepth, interacted, prevId, nextId) 1227 + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL) 1228 + `).run( 1229 + itemVisitId, 1230 + itemId, 1231 + visit.timestamp, 1232 + visit.duration || 0, 1233 + visit.source || 'direct', 1234 + visit.sourceId || '', 1235 + visit.windowType || 'main', 1236 + visit.metadata || '{}', 1237 + visit.scrollDepth || 0, 1238 + visit.interacted || 0 1239 + ); 1240 + 1241 + migratedCount++; 1242 + } 1243 + 1244 + // Second pass: update chaining (prevId/nextId) 1245 + for (const visit of allVisits) { 1246 + const newVisitId = visitIdMap[visit.id]; 1247 + if (!newVisitId) continue; 1248 + 1249 + const newPrevId = visit.prevId ? visitIdMap[visit.prevId] : null; 1250 + const newNextId = visit.nextId ? visitIdMap[visit.nextId] : null; 1251 + 1252 + if (newPrevId || newNextId) { 1253 + db.prepare('UPDATE item_visits SET prevId = ?, nextId = ? WHERE id = ?').run( 1254 + newPrevId || null, 1255 + newNextId || null, 1256 + newVisitId 1257 + ); 1258 + } 1259 + } 1260 + 1261 + // Calculate initial frecency scores for all URL items 1262 + const urlItems = db.prepare('SELECT id FROM items WHERE type = ? AND deletedAt = 0').all('url') as { id: string }[]; 1263 + for (const item of urlItems) { 1264 + const visits = db.prepare('SELECT timestamp, interacted, source FROM item_visits WHERE itemId = ?').all(item.id) as Array<{ timestamp: number; interacted: number; source: string }>; 1265 + const frecencyScore = calculateItemFrecency(visits); 1266 + db.prepare('UPDATE items SET frecencyScore = ? WHERE id = ?').run(frecencyScore, item.id); 1267 + } 1268 + 1269 + // Mark migration as complete 1270 + db.prepare('INSERT OR REPLACE INTO migrations (id, status, completedAt) VALUES (?, ?, ?)').run(MIGRATION_ID, 'complete', Date.now()); 1271 + DEBUG && console.log('main', `Migrated ${migratedCount} visits to item_visits, skipped ${skippedCount}, calculated frecency for ${urlItems.length} items`); 1272 + } 1273 + 896 1274 // ==================== Version Check ==================== 897 1275 898 1276 /** ··· 1619 1997 conditions.push('archived = ?'); 1620 1998 values.push(filter.archived); 1621 1999 } 2000 + if (filter.domain) { 2001 + conditions.push('domain = ?'); 2002 + values.push(filter.domain); 2003 + } 2004 + if (filter.search) { 2005 + // Search in content (URL), title, and domain 2006 + conditions.push('(content LIKE ? OR title LIKE ? OR domain LIKE ?)'); 2007 + const searchPattern = `%${filter.search}%`; 2008 + values.push(searchPattern, searchPattern, searchPattern); 2009 + } 1622 2010 1623 2011 const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; 1624 - const orderBy = filter.sortBy === 'updated' ? 'updatedAt DESC' : 'createdAt DESC'; 2012 + 2013 + // Support multiple sort options 2014 + let orderBy: string; 2015 + switch (filter.sortBy) { 2016 + case 'frecency': 2017 + orderBy = 'frecencyScore DESC, lastVisitAt DESC'; 2018 + break; 2019 + case 'lastVisit': 2020 + orderBy = 'lastVisitAt DESC'; 2021 + break; 2022 + case 'visitCount': 2023 + orderBy = 'visitCount DESC'; 2024 + break; 2025 + case 'updated': 2026 + orderBy = 'updatedAt DESC'; 2027 + break; 2028 + case 'created': 2029 + default: 2030 + orderBy = 'createdAt DESC'; 2031 + } 2032 + 1625 2033 const limit = filter.limit ? `LIMIT ${filter.limit}` : ''; 1626 2034 1627 2035 return getDb().prepare( ··· 1695 2103 WHERE it.tagId = ? AND i.deletedAt = 0 1696 2104 `).all(tagId) as Item[]; 1697 2105 } 2106 + 2107 + // ==================== Item Visit Operations ==================== 2108 + 2109 + /** 2110 + * Record a visit to an item. Updates visit stats and frecency score. 2111 + */ 2112 + export function recordItemVisit(itemId: string, options: ItemVisitOptions = {}): { id: string } { 2113 + const visitId = generateId('item_visit'); 2114 + const timestamp = options.timestamp || now(); 2115 + const d = getDb(); 2116 + 2117 + // Find the most recent item visit for chaining 2118 + const prevVisit = d.prepare('SELECT id FROM item_visits ORDER BY timestamp DESC LIMIT 1').get() as { id: string } | undefined; 2119 + const prevId = prevVisit ? prevVisit.id : null; 2120 + 2121 + d.prepare(` 2122 + INSERT INTO item_visits (id, itemId, timestamp, duration, source, sourceId, windowType, metadata, scrollDepth, interacted, prevId, nextId) 2123 + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL) 2124 + `).run( 2125 + visitId, 2126 + itemId, 2127 + timestamp, 2128 + options.duration || 0, 2129 + options.source || 'direct', 2130 + options.sourceId || '', 2131 + options.windowType || 'main', 2132 + options.metadata || '{}', 2133 + options.scrollDepth || 0, 2134 + options.interacted || 0, 2135 + prevId 2136 + ); 2137 + 2138 + // Update nextId on the previous visit 2139 + if (prevId) { 2140 + d.prepare('UPDATE item_visits SET nextId = ? WHERE id = ?').run(visitId, prevId); 2141 + } 2142 + 2143 + // Update item visit stats and recalculate frecency 2144 + updateItemVisitStats(itemId); 2145 + 2146 + return { id: visitId }; 2147 + } 2148 + 2149 + /** 2150 + * Get visits for an item with optional filters 2151 + */ 2152 + export function getItemVisits(itemId: string, filter: ItemVisitFilter = {}): ItemVisit[] { 2153 + let sql = 'SELECT * FROM item_visits WHERE itemId = ?'; 2154 + const params: (string | number)[] = [itemId]; 2155 + 2156 + if (filter.source) { 2157 + sql += ' AND source = ?'; 2158 + params.push(filter.source); 2159 + } 2160 + if (filter.since) { 2161 + sql += ' AND timestamp >= ?'; 2162 + params.push(filter.since); 2163 + } 2164 + if (filter.until) { 2165 + sql += ' AND timestamp <= ?'; 2166 + params.push(filter.until); 2167 + } 2168 + 2169 + sql += ' ORDER BY timestamp DESC'; 2170 + 2171 + if (filter.limit) { 2172 + sql += ' LIMIT ?'; 2173 + params.push(filter.limit); 2174 + } 2175 + 2176 + return getDb().prepare(sql).all(...params) as ItemVisit[]; 2177 + } 2178 + 2179 + /** 2180 + * Query item visits across all items 2181 + */ 2182 + export function queryItemVisits(filter: ItemVisitFilter = {}): ItemVisit[] { 2183 + let sql = 'SELECT * FROM item_visits WHERE 1=1'; 2184 + const params: (string | number)[] = []; 2185 + 2186 + if (filter.itemId) { 2187 + sql += ' AND itemId = ?'; 2188 + params.push(filter.itemId); 2189 + } 2190 + if (filter.source) { 2191 + sql += ' AND source = ?'; 2192 + params.push(filter.source); 2193 + } 2194 + if (filter.since) { 2195 + sql += ' AND timestamp >= ?'; 2196 + params.push(filter.since); 2197 + } 2198 + if (filter.until) { 2199 + sql += ' AND timestamp <= ?'; 2200 + params.push(filter.until); 2201 + } 2202 + 2203 + sql += ' ORDER BY timestamp DESC'; 2204 + 2205 + if (filter.limit) { 2206 + sql += ' LIMIT ?'; 2207 + params.push(filter.limit); 2208 + } 2209 + 2210 + return getDb().prepare(sql).all(...params) as ItemVisit[]; 2211 + } 2212 + 2213 + /** 2214 + * Update visit count, lastVisitAt, and frecency score for an item 2215 + */ 2216 + export function updateItemVisitStats(itemId: string): void { 2217 + const d = getDb(); 2218 + const timestamp = now(); 2219 + 2220 + // Get all visits for this item 2221 + const visits = d.prepare('SELECT timestamp, interacted, source FROM item_visits WHERE itemId = ?').all(itemId) as Array<{ timestamp: number; interacted: number; source: string }>; 2222 + 2223 + const visitCount = visits.length; 2224 + const lastVisitAt = visits.length > 0 ? Math.max(...visits.map(v => v.timestamp)) : 0; 2225 + const frecencyScore = calculateItemFrecency(visits); 2226 + 2227 + d.prepare('UPDATE items SET visitCount = ?, lastVisitAt = ?, frecencyScore = ?, updatedAt = ? WHERE id = ?').run( 2228 + visitCount, 2229 + lastVisitAt, 2230 + frecencyScore, 2231 + timestamp, 2232 + itemId 2233 + ); 2234 + } 2235 + 2236 + /** 2237 + * Unified entry point for tracking navigation. 2238 + * Finds or creates an item for the URL, then records a visit. 2239 + * This is the main API for tracking page loads. 2240 + */ 2241 + export function trackNavigation(uri: string, options: { 2242 + source?: string; 2243 + sourceId?: string; 2244 + windowType?: string; 2245 + title?: string; 2246 + favicon?: string; 2247 + interacted?: number; 2248 + } = {}): { visitId: string; itemId: string; created: boolean } { 2249 + const normalizedUri = normalizeUrl(uri); 2250 + const parsed = parseUrl(normalizedUri); 2251 + const d = getDb(); 2252 + let created = false; 2253 + 2254 + // Find existing item by URL 2255 + const existing = d.prepare('SELECT id FROM items WHERE type = ? AND content = ? AND deletedAt = 0').get('url', normalizedUri) as { id: string } | undefined; 2256 + 2257 + let itemId: string; 2258 + if (existing) { 2259 + itemId = existing.id; 2260 + // Update title/favicon if provided and item is missing them 2261 + const updates: string[] = []; 2262 + const values: unknown[] = []; 2263 + 2264 + if (options.title) { 2265 + updates.push('title = CASE WHEN title = \'\' OR title IS NULL THEN ? ELSE title END'); 2266 + values.push(options.title); 2267 + } 2268 + if (options.favicon) { 2269 + updates.push('favicon = CASE WHEN favicon = \'\' OR favicon IS NULL THEN ? ELSE favicon END'); 2270 + values.push(options.favicon); 2271 + } 2272 + 2273 + if (updates.length > 0) { 2274 + values.push(itemId); 2275 + d.prepare(`UPDATE items SET ${updates.join(', ')} WHERE id = ?`).run(...values); 2276 + } 2277 + } else { 2278 + // Create new item for this URL 2279 + itemId = generateId('item'); 2280 + const timestamp = now(); 2281 + created = true; 2282 + 2283 + const metadata: Record<string, unknown> = {}; 2284 + if (options.title) metadata.title = options.title; 2285 + if (options.favicon) metadata.favicon = options.favicon; 2286 + 2287 + d.prepare(` 2288 + INSERT INTO items (id, type, content, mimeType, metadata, syncId, syncSource, createdAt, updatedAt, deletedAt, starred, archived, visitCount, lastVisitAt, frecencyScore, title, domain, favicon) 2289 + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0, ?, ?, ?) 2290 + `).run( 2291 + itemId, 2292 + 'url', 2293 + normalizedUri, 2294 + 'text/html', 2295 + JSON.stringify(metadata), 2296 + '', 2297 + '', 2298 + timestamp, 2299 + timestamp, 2300 + options.title || '', 2301 + parsed.domain, 2302 + options.favicon || '' 2303 + ); 2304 + } 2305 + 2306 + // Record the visit 2307 + const visit = recordItemVisit(itemId, { 2308 + source: options.source || 'window', 2309 + sourceId: options.sourceId || '', 2310 + windowType: options.windowType || 'main', 2311 + interacted: options.interacted || 0, 2312 + }); 2313 + 2314 + // Also track in legacy visits table for backward compatibility 2315 + // (This ensures old code that queries addresses/visits still works) 2316 + const addressResult = d.prepare('SELECT id FROM addresses WHERE uri = ?').get(normalizedUri) as { id: string } | undefined; 2317 + if (addressResult) { 2318 + addVisit(addressResult.id, { 2319 + source: options.source || 'window', 2320 + sourceId: options.sourceId || '', 2321 + windowType: options.windowType || 'main', 2322 + }); 2323 + } 2324 + 2325 + return { visitId: visit.id, itemId, created }; 2326 + } 2327 + 2328 + /** 2329 + * Query items by frecency, optimized for history/omnibox use cases. 2330 + * Returns URL items sorted by frecency score. 2331 + */ 2332 + export function queryItemsByFrecency(filter: { 2333 + search?: string; 2334 + domain?: string; 2335 + limit?: number; 2336 + since?: number; 2337 + } = {}): Item[] { 2338 + const conditions: string[] = ['type = ?', 'deletedAt = 0']; 2339 + const values: unknown[] = ['url']; 2340 + 2341 + if (filter.search) { 2342 + conditions.push('(content LIKE ? OR title LIKE ? OR domain LIKE ?)'); 2343 + const searchPattern = `%${filter.search}%`; 2344 + values.push(searchPattern, searchPattern, searchPattern); 2345 + } 2346 + if (filter.domain) { 2347 + conditions.push('domain = ?'); 2348 + values.push(filter.domain); 2349 + } 2350 + if (filter.since) { 2351 + conditions.push('lastVisitAt >= ?'); 2352 + values.push(filter.since); 2353 + } 2354 + 2355 + const whereClause = `WHERE ${conditions.join(' AND ')}`; 2356 + const limit = filter.limit ? `LIMIT ${filter.limit}` : 'LIMIT 50'; 2357 + 2358 + return getDb().prepare( 2359 + `SELECT * FROM items ${whereClause} ORDER BY frecencyScore DESC, lastVisitAt DESC ${limit}` 2360 + ).all(...values) as Item[]; 2361 + } 2362 + 2363 + /** 2364 + * Backward compatibility: query addresses and transform to Address shape. 2365 + * This wraps queryItems for code that still uses the old address API. 2366 + */ 2367 + export function queryAddressesCompat(filter: AddressFilter = {}): Address[] { 2368 + // Map address filter to item filter 2369 + const itemFilter: ItemFilter = { 2370 + type: 'url', 2371 + }; 2372 + 2373 + if (filter.starred !== undefined) { 2374 + itemFilter.starred = filter.starred; 2375 + } 2376 + if (filter.domain) { 2377 + itemFilter.domain = filter.domain; 2378 + } 2379 + if (filter.limit) { 2380 + itemFilter.limit = filter.limit; 2381 + } 2382 + 2383 + // Map sortBy 2384 + switch (filter.sortBy) { 2385 + case 'lastVisit': 2386 + itemFilter.sortBy = 'lastVisit'; 2387 + break; 2388 + case 'visitCount': 2389 + itemFilter.sortBy = 'visitCount'; 2390 + break; 2391 + case 'created': 2392 + default: 2393 + itemFilter.sortBy = 'created'; 2394 + } 2395 + 2396 + const items = queryItems(itemFilter); 2397 + 2398 + // Transform items to Address shape 2399 + return items.map(item => { 2400 + const parsed = parseUrl(item.content || ''); 2401 + let metadata: Record<string, unknown> = {}; 2402 + try { 2403 + metadata = typeof item.metadata === 'string' ? JSON.parse(item.metadata) : item.metadata || {}; 2404 + } catch { /* ignore */ } 2405 + 2406 + return { 2407 + id: item.id, 2408 + uri: item.content || '', 2409 + protocol: parsed.protocol, 2410 + domain: item.domain || parsed.domain, 2411 + path: parsed.path, 2412 + title: item.title || (metadata.title as string) || '', 2413 + mimeType: item.mimeType || 'text/html', 2414 + favicon: item.favicon || (metadata.favicon as string) || '', 2415 + description: (metadata.description as string) || '', 2416 + tags: '', 2417 + metadata: item.metadata, 2418 + createdAt: item.createdAt, 2419 + updatedAt: item.updatedAt, 2420 + lastVisitAt: item.lastVisitAt, 2421 + visitCount: item.visitCount, 2422 + starred: item.starred, 2423 + archived: item.archived, 2424 + }; 2425 + }); 2426 + }
+57
backend/electron/ipc.ts
··· 43 43 untagItem, 44 44 getItemTags, 45 45 getItemsByTag, 46 + // Item visit operations (URL history unification) 47 + recordItemVisit, 48 + getItemVisits, 49 + queryItemVisits, 50 + trackNavigation, 51 + queryItemsByFrecency, 46 52 // History operations 47 53 trackWindowLoad, 48 54 getHistory, ··· 537 543 ipcMain.handle('datastore-get-history', async (ev, data = {}) => { 538 544 try { 539 545 const result = getHistory(data.filter); 546 + return { success: true, data: result }; 547 + } catch (error) { 548 + const message = error instanceof Error ? error.message : String(error); 549 + return { success: false, error: message }; 550 + } 551 + }); 552 + 553 + // Item visit operations (URL history unification) 554 + ipcMain.handle('datastore-record-item-visit', async (ev, data) => { 555 + try { 556 + const result = recordItemVisit(data.itemId, data.options); 557 + return { success: true, data: result }; 558 + } catch (error) { 559 + const message = error instanceof Error ? error.message : String(error); 560 + return { success: false, error: message }; 561 + } 562 + }); 563 + 564 + ipcMain.handle('datastore-get-item-visits', async (ev, data) => { 565 + try { 566 + const result = getItemVisits(data.itemId, data.filter); 567 + return { success: true, data: result }; 568 + } catch (error) { 569 + const message = error instanceof Error ? error.message : String(error); 570 + return { success: false, error: message }; 571 + } 572 + }); 573 + 574 + ipcMain.handle('datastore-query-item-visits', async (ev, data = {}) => { 575 + try { 576 + const result = queryItemVisits(data.filter); 577 + return { success: true, data: result }; 578 + } catch (error) { 579 + const message = error instanceof Error ? error.message : String(error); 580 + return { success: false, error: message }; 581 + } 582 + }); 583 + 584 + ipcMain.handle('datastore-track-navigation', async (ev, data) => { 585 + try { 586 + const result = trackNavigation(data.uri, data.options); 587 + return { success: true, data: result }; 588 + } catch (error) { 589 + const message = error instanceof Error ? error.message : String(error); 590 + return { success: false, error: message }; 591 + } 592 + }); 593 + 594 + ipcMain.handle('datastore-query-items-by-frecency', async (ev, data = {}) => { 595 + try { 596 + const result = queryItemsByFrecency(data.filter); 540 597 return { success: true, data: result }; 541 598 } catch (error) { 542 599 const message = error instanceof Error ? error.message : String(error);
+60 -15
backend/extension/history.js
··· 8 8 * Behind a test feature toggle (peek_history_enabled). 9 9 */ 10 10 11 - import { addItem, queryItems, getOrCreateTag, tagItem, updateItem, getItemsByTag } from './datastore.js'; 11 + import { addItem, queryItems, getOrCreateTag, tagItem, updateItem, getItemsByTag, recordItemVisit } from './datastore.js'; 12 12 13 13 const CONFIG_KEY = 'peek_history_enabled'; 14 14 const HISTORY_TAG = 'from:history'; ··· 82 82 83 83 /** 84 84 * Add a new history item or update an existing one with fresh visit data. 85 + * Also records item visits for frecency calculation. 85 86 * Returns 'imported' | 'updated' | 'skipped'. 86 87 */ 87 88 async function addOrUpdateHistoryItem(url, historyItem, visits, existingUrlMap) { ··· 89 90 90 91 const metadata = buildHistoryMetadata(historyItem, visits); 91 92 const existing = existingUrlMap.get(url); 93 + let itemId; 92 94 93 95 if (existing) { 96 + itemId = existing.id; 94 97 // Update existing item with fresh metadata 95 98 await updateItem(existing.id, { metadata }); 96 99 // Tag existing item so it's counted in history stats ··· 98 101 if (tagResult.success) { 99 102 await tagItem(existing.id, tagResult.data.tag.id); 100 103 } 101 - return 'updated'; 104 + } else { 105 + // Add new item 106 + const result = await addItem('url', { 107 + content: url, 108 + metadata, 109 + syncSource: 'history', 110 + }); 111 + 112 + if (result.success) { 113 + itemId = result.data.id; 114 + const tagResult = await getOrCreateTag(HISTORY_TAG); 115 + if (tagResult.success) { 116 + await tagItem(result.data.id, tagResult.data.tag.id); 117 + } 118 + existingUrlMap.set(url, { id: result.data.id, content: url }); 119 + } 102 120 } 103 121 104 - // Add new item 105 - const result = await addItem('url', { 106 - content: url, 107 - metadata, 108 - syncSource: 'history', 109 - }); 110 - 111 - if (result.success) { 112 - const tagResult = await getOrCreateTag(HISTORY_TAG); 113 - if (tagResult.success) { 114 - await tagItem(result.data.id, tagResult.data.tag.id); 122 + // Record item visits for frecency calculation 123 + // This populates the item_visits table which is used for frecency scoring 124 + if (itemId && visits && visits.length > 0) { 125 + for (const visit of visits) { 126 + // Map browser transition types to our source types 127 + const source = mapTransitionToSource(visit.transition); 128 + await recordItemVisit(itemId, { 129 + timestamp: visit.visitTime, 130 + source, 131 + sourceId: visit.referringVisitId ? String(visit.referringVisitId) : '', 132 + // Typed visits are considered "interacted" as they show user intent 133 + interacted: visit.transition === 'typed' ? 1 : 0, 134 + }); 115 135 } 116 - existingUrlMap.set(url, { id: result.data.id, content: url }); 117 136 } 118 137 119 - return 'imported'; 138 + return existing ? 'updated' : 'imported'; 139 + } 140 + 141 + /** 142 + * Map Chrome's transition types to our source types 143 + */ 144 + function mapTransitionToSource(transition) { 145 + switch (transition) { 146 + case 'link': 147 + return 'link'; 148 + case 'typed': 149 + return 'direct'; 150 + case 'auto_bookmark': 151 + return 'bookmark'; 152 + case 'auto_subframe': 153 + case 'manual_subframe': 154 + return 'frame'; 155 + case 'generated': 156 + case 'auto_toplevel': 157 + return 'generated'; 158 + case 'form_submit': 159 + return 'form'; 160 + case 'reload': 161 + return 'reload'; 162 + default: 163 + return 'other'; 164 + } 120 165 } 121 166 122 167 // ==================== Import ====================
+74 -3
backend/types/index.ts
··· 106 106 archived: number; 107 107 visitCount: number; 108 108 lastVisitAt: number; 109 + // Local-only columns for URL history unification (not synced) 110 + frecencyScore: number; 111 + title: string; 112 + domain: string; 113 + favicon: string; 114 + } 115 + 116 + // Visit record for item_visits table (local-only, not synced) 117 + export interface ItemVisit { 118 + id: string; 119 + itemId: string; 120 + timestamp: number; 121 + duration: number; 122 + source: string; 123 + sourceId: string; 124 + windowType: string; 125 + metadata: string; 126 + scrollDepth: number; 127 + interacted: number; 128 + prevId: string | null; 129 + nextId: string | null; 130 + } 131 + 132 + // Group for organizing items (future-proofing) 133 + export interface ItemGroup { 134 + id: string; 135 + name: string; 136 + description: string; 137 + type: string; 138 + query: string; 139 + metadata: string; 140 + createdAt: number; 141 + updatedAt: number; 142 + deletedAt: number; 143 + } 144 + 145 + export interface ItemGroupMember { 146 + id: string; 147 + groupId: string; 148 + itemId: string; 149 + position: number; 150 + createdAt: number; 109 151 } 110 152 111 153 export interface ItemTag { ··· 239 281 archived?: number; 240 282 includeDeleted?: boolean; 241 283 limit?: number; 242 - sortBy?: 'created' | 'updated'; 284 + sortBy?: 'created' | 'updated' | 'frecency' | 'lastVisit' | 'visitCount'; 285 + domain?: string; 286 + search?: string; 287 + } 288 + 289 + export interface ItemVisitFilter { 290 + itemId?: string; 291 + source?: string; 292 + since?: number; 293 + until?: number; 294 + limit?: number; 295 + } 296 + 297 + export interface ItemVisitOptions { 298 + timestamp?: number; 299 + duration?: number; 300 + source?: string; 301 + sourceId?: string; 302 + windowType?: string; 303 + metadata?: string; 304 + scrollDepth?: number; 305 + interacted?: number; 243 306 } 244 307 245 308 // ==================== Table Names ==================== ··· 257 320 | 'extension_settings' 258 321 | 'migrations' 259 322 | 'items' 260 - | 'item_tags'; 323 + | 'item_tags' 324 + | 'item_visits' 325 + | 'item_groups' 326 + | 'item_group_members' 327 + | 'settings'; 261 328 262 329 export const tableNames: TableName[] = [ 263 330 'addresses', ··· 272 339 'extension_settings', 273 340 'migrations', 274 341 'items', 275 - 'item_tags' 342 + 'item_tags', 343 + 'item_visits', 344 + 'item_groups', 345 + 'item_group_members', 346 + 'settings' 276 347 ]; 277 348 278 349 // ==================== Sync Types ====================