Unify links, references, and url_mapping JSON files into single structure

+28 -25

ARCH.md

··· 157 157 git-store/ 158 158 ├── index.json # User directory index 159 159 ├── duplicates.json # Manual curation of duplicate entries 160 - ├── links.json # All outbound links categorized by type 161 - ├── references.json # Cross-reference index for threading 160 + ├── links.json # Unified links, references, and mapping data 162 161 ├── user1/ 163 162 │ ├── entry_id_1.json # Sanitized entry files 164 163 │ ├── entry_id_2.json ··· 403 402 404 403 ### Data Structures 405 404 406 - #### links.json Format 405 + #### links.json Format (Unified Structure) 407 406 ```json 408 407 { 409 - "links": [ 410 - { 411 - "url": "https://example.com/post/123", 412 - "entry_id": "https://blog.user.com/entry/456", 413 - "username": "user1", 414 - "context": "As mentioned in this post...", 415 - "category": "user", 408 + "links": { 409 + "https://example.com/post/123": { 410 + "referencing_entries": ["https://blog.user.com/entry/456"], 411 + "is_tracked_post": true, 416 412 "target_username": "user2" 413 + }, 414 + "https://external-site.com/article": { 415 + "referencing_entries": ["https://blog.user.com/entry/789"], 416 + "is_tracked_post": false 417 417 } 418 - ], 419 - "categories": { 420 - "internal": 1234, 421 - "user": 456, 422 - "unknown": 7890 423 418 }, 424 - "user_domains": { 425 - "user1": ["blog.user.com", "user.com"], 426 - "user2": ["example.com"] 427 - } 428 - } 429 - ``` 430 - 431 - #### references.json Format 432 - ```json 433 - { 419 + "reverse_mapping": { 420 + "https://blog.user.com/entry/456": ["https://example.com/post/123"], 421 + "https://blog.user.com/entry/789": ["https://external-site.com/article"] 422 + }, 434 423 "references": [ 435 424 { 436 425 "source_entry_id": "https://blog.user.com/entry/456", ··· 447 436 } 448 437 } 449 438 ``` 439 + 440 + This unified structure eliminates duplication by: 441 + - Storing each URL only once with metadata flags 442 + - Including all link data, reference data, and mappings in one file 443 + - Using `is_tracked_post` to identify internal vs external links 444 + - Providing bidirectional mappings for efficient queries 445 + 446 + ### Unified Structure Benefits 447 + 448 + - **Eliminates Duplication**: Each URL appears only once with metadata 449 + - **Single Source of Truth**: All link-related data in one file 450 + - **Efficient Queries**: Fast lookups for both directions (URL→entries, entry→URLs) 451 + - **Atomic Updates**: All link data changes together 452 + - **Reduced I/O**: Fewer file operations 450 453 451 454 ### Implementation Benefits 452 455

+42 -11

src/thicket/cli/commands/index_cmd.py

··· 35 35 None, 36 36 "--output", 37 37 "-o", 38 - help="Path to output index file (default: references.json in git store)", 38 + help="Path to output index file (default: updates links.json in git store)", 39 39 ), 40 40 verbose: bool = typer.Option( 41 41 False, ··· 49 49 This command analyzes all blog entries to detect cross-references between 50 50 different blogs, creating an index that can be used to build threaded 51 51 views of related content. 52 + 53 + Updates the unified links.json file with reference data. 52 54 """ 53 55 try: 54 56 # Load configuration ··· 154 156 if output_file: 155 157 output_path = output_file 156 158 else: 157 - output_path = config.git_store / "references.json" 159 + output_path = config.git_store / "links.json" 160 + 161 + # Load existing links data or create new structure 162 + if output_path.exists() and not output_file: 163 + # Load existing unified structure 164 + with open(output_path) as f: 165 + existing_data = json.load(f) 166 + else: 167 + # Create new structure 168 + existing_data = { 169 + "links": {}, 170 + "reverse_mapping": {}, 171 + "user_domains": {} 172 + } 173 + 174 + # Update with reference data 175 + existing_data["references"] = ref_index.to_dict()["references"] 176 + existing_data["user_domains"] = {k: list(v) for k, v in user_domains.items()} 158 177 159 - # Save reference index 178 + # Save updated structure 160 179 with open(output_path, "w") as f: 161 - json.dump(ref_index.to_dict(), f, indent=2, default=str) 180 + json.dump(existing_data, f, indent=2, default=str) 162 181 163 182 # Show summary 164 183 if not get_tsv_mode(): ··· 248 267 None, 249 268 "--index", 250 269 "-i", 251 - help="Path to reference index file (default: references.json in git store)", 270 + help="Path to reference index file (default: links.json in git store)", 252 271 ), 253 272 username: Optional[str] = typer.Option( 254 273 None, ··· 274 293 This command uses the reference index to show which blog entries 275 294 are connected through cross-references, creating an email-style 276 295 threaded view of the conversation. 296 + 297 + Reads reference data from the unified links.json file. 277 298 """ 278 299 try: 279 300 # Load configuration ··· 283 304 if index_file: 284 305 index_path = index_file 285 306 else: 286 - index_path = config.git_store / "references.json" 307 + index_path = config.git_store / "links.json" 287 308 288 309 if not index_path.exists(): 289 - console.print(f"[red]Reference index not found: {index_path}[/red]") 290 - console.print("Run 'thicket index' first to build the reference index") 310 + console.print(f"[red]Links file not found: {index_path}[/red]") 311 + console.print("Run 'thicket links' and 'thicket index' first to build the reference index") 291 312 raise typer.Exit(1) 292 313 293 - # Load reference index 314 + # Load unified data 294 315 with open(index_path) as f: 295 - index_data = json.load(f) 316 + unified_data = json.load(f) 317 + 318 + # Check if references exist in the unified structure 319 + if "references" not in unified_data: 320 + console.print(f"[red]No references found in {index_path}[/red]") 321 + console.print("Run 'thicket index' first to build the reference index") 322 + raise typer.Exit(1) 296 323 297 - ref_index = ReferenceIndex.from_dict(index_data) 324 + # Extract reference data and reconstruct ReferenceIndex 325 + ref_index = ReferenceIndex.from_dict({ 326 + "references": unified_data["references"], 327 + "user_domains": unified_data.get("user_domains", {}) 328 + }) 298 329 299 330 # Initialize Git store to get entry details 300 331 git_store = GitStore(config.git_store)

+12 -6

src/thicket/cli/commands/info_cmd.py

··· 106 106 raise typer.Exit(1) 107 107 108 108 # Load reference index if available 109 - references_path = config.git_store / "references.json" 109 + links_path = config.git_store / "links.json" 110 110 ref_index = None 111 - if references_path.exists(): 112 - with open(references_path) as f: 113 - index_data = json.load(f) 114 - ref_index = ReferenceIndex.from_dict(index_data) 111 + if links_path.exists(): 112 + with open(links_path) as f: 113 + unified_data = json.load(f) 114 + 115 + # Check if references exist in the unified structure 116 + if "references" in unified_data: 117 + ref_index = ReferenceIndex.from_dict({ 118 + "references": unified_data["references"], 119 + "user_domains": unified_data.get("user_domains", {}) 120 + }) 115 121 116 122 # Display information 117 123 if get_tsv_mode(): ··· 122 128 if ref_index: 123 129 _display_link_info(entry, found_username, ref_index) 124 130 else: 125 - console.print("\n[yellow]No reference index found. Run 'thicket index' to build cross-reference data.[/yellow]") 131 + console.print("\n[yellow]No reference index found. Run 'thicket links' and 'thicket index' to build cross-reference data.[/yellow]") 126 132 127 133 # Optionally display content 128 134 if show_content and entry.content:

+45 -44

src/thicket/cli/commands/links_cmd.py

··· 145 145 None, 146 146 "--output", 147 147 "-o", 148 - help="Path to output links file (default: links.json in git store)", 149 - ), 150 - mapping_file: Optional[Path] = typer.Option( 151 - None, 152 - "--mapping", 153 - "-m", 154 - help="Path to output URL <-> atom ID mapping file (default: url_mapping.json in git store)", 148 + help="Path to output unified links file (default: links.json in git store)", 155 149 ), 156 150 verbose: bool = typer.Option( 157 151 False, ··· 165 159 This command analyzes all blog entries to extract outbound links, 166 160 resolve them properly with respect to the feed's base URL, and 167 161 categorize them as internal, user, or unknown links. 162 + 163 + Creates a unified links.json file containing all link data. 168 164 """ 169 165 try: 170 166 # Load configuration ··· 290 286 if verbose and entry_links: 291 287 console.print(f" Found {len(entry_links)} links in {username}:{entry.title[:50]}...") 292 288 293 - # Determine output paths 289 + # Determine output path 294 290 if output_file: 295 291 output_path = output_file 296 292 else: 297 293 output_path = config.git_store / "links.json" 298 - 299 - if mapping_file: 300 - mapping_path = mapping_file 301 - else: 302 - mapping_path = config.git_store / "url_mapping.json" 303 294 304 295 # Save all extracted links (not just filtered ones) 305 296 if verbose: ··· 324 315 if hasattr(link, 'href') and link.href: 325 316 registered_urls.add(str(link.href)) 326 317 327 - # Create filtered version for URL mapping (only links to registered posts) 328 - filtered_link_dict = {} 329 - filtered_reverse_dict = {} 318 + # Build unified structure with metadata 319 + unified_links = {} 320 + reverse_mapping = {} 330 321 331 322 for url, entry_ids in link_dict.items(): 332 - if url in registered_urls: 333 - filtered_link_dict[url] = entry_ids 334 - 335 - # Also update reverse mapping 336 - for entry_id in entry_ids: 337 - if entry_id not in filtered_reverse_dict: 338 - filtered_reverse_dict[entry_id] = [] 339 - if url not in filtered_reverse_dict[entry_id]: 340 - filtered_reverse_dict[entry_id].append(url) 323 + is_tracked = url in registered_urls 324 + target_username = None 325 + 326 + # Find target username if this is a tracked post 327 + if is_tracked: 328 + for username in users: 329 + user_domains_set = {domain for domain in user_domains.get(username, [])} 330 + if any(domain in url for domain in user_domains_set): 331 + target_username = username 332 + break 333 + 334 + unified_links[url] = { 335 + "referencing_entries": entry_ids, 336 + "is_tracked_post": is_tracked 337 + } 338 + 339 + if target_username: 340 + unified_links[url]["target_username"] = target_username 341 + 342 + # Build reverse mapping 343 + for entry_id in entry_ids: 344 + if entry_id not in reverse_mapping: 345 + reverse_mapping[entry_id] = [] 346 + if url not in reverse_mapping[entry_id]: 347 + reverse_mapping[entry_id].append(url) 341 348 342 - # Use all links for main output, not filtered ones 343 - output_data = link_dict 349 + # Create unified output data 350 + output_data = { 351 + "links": unified_links, 352 + "reverse_mapping": reverse_mapping, 353 + "user_domains": {k: list(v) for k, v in user_domains.items()} 354 + } 344 355 345 356 if verbose: 346 357 console.print(f"Found {len(registered_urls)} registered post URLs") 347 - console.print(f"Found {len(link_dict)} total links, {len(filtered_link_dict)} links to registered posts") 358 + console.print(f"Found {len(link_dict)} total links, {sum(1 for link in unified_links.values() if link['is_tracked_post'])} tracked posts") 348 359 349 - # Save links data (URL -> atom ID mapping, all links) 360 + # Save unified data 350 361 with open(output_path, "w") as f: 351 362 json.dump(output_data, f, indent=2, default=str) 352 - 353 - # Save bidirectional mapping file (filtered) 354 - mapping_data = { 355 - "url_to_atom": filtered_link_dict, 356 - "atom_to_urls": filtered_reverse_dict 357 - } 358 - 359 - with open(mapping_path, "w") as f: 360 - json.dump(mapping_data, f, indent=2, default=str) 361 363 362 364 # Show summary 363 365 if not get_tsv_mode(): ··· 370 372 print(f"User\t{len(link_categories['user'])}\tLinks to other tracked users") 371 373 print(f"Unknown\t{len(link_categories['unknown'])}\tLinks to external sites") 372 374 print(f"Total Extracted\t{len(all_links)}\tAll extracted links") 373 - print(f"Saved to Output\t{len(output_data)}\tLinks saved to output file") 374 - print(f"Cross-references\t{len(filtered_link_dict)}\tLinks to registered posts only") 375 + print(f"Saved to Output\t{len(output_data['links'])}\tLinks saved to output file") 376 + print(f"Cross-references\t{sum(1 for link in unified_links.values() if link['is_tracked_post'])}\tLinks to registered posts only") 375 377 else: 376 378 table = Table(title="Links Summary") 377 379 table.add_column("Category", style="cyan") ··· 382 384 table.add_row("User", str(len(link_categories["user"])), "Links to other tracked users") 383 385 table.add_row("Unknown", str(len(link_categories["unknown"])), "Links to external sites") 384 386 table.add_row("Total Extracted", str(len(all_links)), "All extracted links") 385 - table.add_row("Saved to Output", str(len(output_data)), "Links saved to output file") 386 - table.add_row("Cross-references", str(len(filtered_link_dict)), "Links to registered posts only") 387 + table.add_row("Saved to Output", str(len(output_data['links'])), "Links saved to output file") 388 + table.add_row("Cross-references", str(sum(1 for link in unified_links.values() if link['is_tracked_post'])), "Links to registered posts only") 387 389 388 390 console.print(table) 389 391 ··· 412 414 console.print(f" {link_pair}: {count} links") 413 415 414 416 if not get_tsv_mode(): 415 - console.print(f"\nLinks output saved to: {output_path}") 416 - console.print(f"URL mapping saved to: {mapping_path}") 417 + console.print(f"\nUnified links data saved to: {output_path}") 417 418 418 419 except Exception as e: 419 420 console.print(f"[red]Error extracting links: {e}[/red]")

Configure Feed

Configure Feed