Remove re-run functionality from import app · solpbc.org/solstone@12d1298

-164

apps/import/_detail.html

··· 67 67 .summary-loading .spinner { display: inline-block; width: 16px; height: 16px; border: 2px solid #ccc; border-top-color: #007bff; border-radius: 50%; animation: spin 0.8s linear infinite; margin-left: 8px; } 68 68 @keyframes spin { to { transform: rotate(360deg); } } 69 69 70 - /* Re-run section */ 71 - .rerun-section { margin-top: 2em; padding: 1em; background: #f0f8ff; border: 1px solid #007bff; border-radius: 4px; } 72 - .rerun-section h3 { margin-top: 0; color: #007bff; } 73 - .rerun-controls { display: flex; align-items: center; gap: 1em; margin-top: 1em; } 74 - .rerun-controls label { font-weight: bold; } 75 - .rerun-controls select { padding: 6px 10px; border: 1px solid #ddd; border-radius: 4px; background: white; min-width: 150px; } 76 - .rerun-controls button { padding: 8px 16px; background: #007bff; color: white; border: none; border-radius: 4px; cursor: pointer; font-weight: bold; } 77 - .rerun-controls button:hover { background: #0056b3; } 78 - .rerun-controls button:disabled { background: #ccc; cursor: not-allowed; opacity: 0.6; } 79 - .rerun-note { font-size: 0.9em; color: #666; margin-top: 0.5em; font-style: italic; } 80 - 81 - /* Progress display */ 82 - .progress-banner { display: none; background: #e7f3ff; border: 1px solid #007bff; border-radius: 4px; padding: 1em; margin: 1em 0; } 83 - .progress-banner.active { display: block; } 84 - .progress-stage { font-weight: bold; color: #007bff; margin-bottom: 0.5em; } 85 - .progress-timing { font-size: 0.9em; color: #666; } 86 - .progress-spinner { display: inline-block; width: 12px; height: 12px; border: 2px solid #007bff; border-top-color: transparent; border-radius: 50%; animation: spin 1s linear infinite; margin-left: 6px; } 87 70 </style> 88 71 89 72 <div class="import-detail-content"> ··· 138 121 </div> 139 122 </div> 140 123 </div> 141 - 142 - <div class="progress-banner" id="progressBanner"> 143 - <div class="progress-stage" id="progressStage">Processing<span class="progress-spinner"></span></div> 144 - <div class="progress-timing" id="progressTiming"></div> 145 - </div> 146 - 147 - <div class="rerun-section" id="rerunSection" style="display:none;"> 148 - <h3>Re-run Import</h3> 149 - <div class="rerun-controls"> 150 - <label for="facetSelect">Facet:</label> 151 - <select id="facetSelect"> 152 - <option value="">None</option> 153 - </select> 154 - <label for="settingInput">Setting:</label> 155 - <input id="settingInput" placeholder="e.g. Jer lunch with Joe" /> 156 - <button id="rerunBtn">Re-run Import</button> 157 - </div> 158 - <div class="rerun-note"> 159 - Re-running will overwrite the existing transcription results with new ones using the selected facet for improved entity recognition. 160 - </div> 161 - </div> 162 124 </div> 163 125 164 126 <script> ··· 229 191 return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]; 230 192 } 231 193 232 - // Format elapsed time 233 - function formatElapsed(ms) { 234 - if (!ms) return '0s'; 235 - const totalSeconds = Math.floor(ms / 1000); 236 - const mins = Math.floor(totalSeconds / 60); 237 - const secs = totalSeconds % 60; 238 - if (mins > 0) { 239 - return `${mins}m ${secs}s`; 240 - } 241 - return `${secs}s`; 242 - } 243 - 244 - // Capitalize stage name 245 - function capitalizeStage(stage) { 246 - if (!stage) return ''; 247 - return stage.charAt(0).toUpperCase() + stage.slice(1); 248 - } 249 - 250 194 // Format timestamp 251 195 function formatTimestamp(timestamp) { 252 196 if (!timestamp) return '-'; ··· 259 203 return timestamp; 260 204 } 261 205 262 - // Global variable to store import data 263 - let importData = null; 264 - const settingInput = document.getElementById('settingInput'); 265 - 266 - // Load facets for dropdown using app framework data 267 - function loadFacets() { 268 - const select = document.getElementById('facetSelect'); 269 - const current = importData?.import_json?.facet || ''; // Use current facet from import 270 - select.innerHTML = '<option value="">None</option>'; 271 - 272 - const facets = window.facetsData || []; 273 - facets.forEach(facet => { 274 - const opt = document.createElement('option'); 275 - opt.value = facet.name; 276 - opt.textContent = facet.title || facet.name; 277 - if (facet.name === current) opt.selected = true; 278 - select.appendChild(opt); 279 - }); 280 - } 281 - 282 - // Show progress banner and listen for importer events 283 - function watchImportProgress(importId) { 284 - const progressBanner = document.getElementById('progressBanner'); 285 - const progressStage = document.getElementById('progressStage'); 286 - const progressTiming = document.getElementById('progressTiming'); 287 - 288 - progressBanner.classList.add('active'); 289 - progressStage.innerHTML = 'Starting<span class="progress-spinner"></span>'; 290 - progressTiming.textContent = ''; 291 - 292 - // Listen for importer events for this import 293 - const cleanup = window.appEvents.listen('importer', msg => { 294 - if (msg.import_id !== importId) return; 295 - 296 - if (msg.event === 'started') { 297 - progressStage.innerHTML = `${capitalizeStage(msg.stage)}<span class="progress-spinner"></span>`; 298 - } else if (msg.event === 'status') { 299 - const stageTime = formatElapsed(msg.stage_elapsed_ms); 300 - const totalTime = formatElapsed(msg.elapsed_ms); 301 - progressStage.innerHTML = `${capitalizeStage(msg.stage)}<span class="progress-spinner"></span>`; 302 - progressTiming.textContent = `${stageTime} (${totalTime} total)`; 303 - } else if (msg.event === 'completed') { 304 - progressStage.innerHTML = '✓ Completed'; 305 - progressTiming.textContent = `Total time: ${formatElapsed(msg.duration_ms)}`; 306 - cleanup(); 307 - // Reload page after successful completion 308 - setTimeout(() => location.reload(), 2000); 309 - } else if (msg.event === 'error') { 310 - progressStage.innerHTML = '✗ Failed'; 311 - progressTiming.textContent = `Error: ${msg.error || 'Unknown error'}`; 312 - cleanup(); 313 - } 314 - }); 315 - } 316 - 317 - // Handle re-run button click 318 - document.getElementById('rerunBtn').addEventListener('click', () => { 319 - const facet = document.getElementById('facetSelect').value; 320 - const setting = settingInput ? settingInput.value.trim() : ''; 321 - 322 - if (confirm('This will re-run the import and overwrite existing transcription results. Continue?')) { 323 - const btn = document.getElementById('rerunBtn'); 324 - btn.disabled = true; 325 - btn.textContent = 'Starting...'; 326 - 327 - // Call the re-run endpoint 328 - fetch('/app/import/api/{{ timestamp }}/rerun', { 329 - method: 'POST', 330 - headers: { 'Content-Type': 'application/json' }, 331 - body: JSON.stringify({ facet: facet, setting: setting }) 332 - }) 333 - .then(r => r.json()) 334 - .then(result => { 335 - if (result.status === 'ok') { 336 - btn.disabled = false; 337 - btn.textContent = 'Re-run Import'; 338 - // Watch progress using importer events with timestamp as import_id 339 - watchImportProgress('{{ timestamp }}'); 340 - } else { 341 - alert('Failed to start re-run: ' + (result.error || 'Unknown error')); 342 - btn.disabled = false; 343 - btn.textContent = 'Re-run Import'; 344 - } 345 - }) 346 - .catch(err => { 347 - alert('Failed to start re-run: ' + err); 348 - btn.disabled = false; 349 - btn.textContent = 'Re-run Import'; 350 - }); 351 - } 352 - }); 353 - 354 206 // Load import details 355 207 fetch('/app/import/api/{{ timestamp }}') 356 208 .then(r => r.json()) 357 209 .then(data => { 358 - importData = data; // Store globally 359 210 // Update header metadata 360 211 const metaDiv = document.getElementById('importMeta'); 361 212 if (data.import_json) { ··· 401 252 if (data.import_json.setting) { 402 253 overviewHtml += `<div class="info-label">Setting:</div><div class="info-value">${data.import_json.setting}</div>`; 403 254 } 404 - if (settingInput) { 405 - settingInput.value = data.import_json.setting || ''; 406 - } 407 255 } 408 256 409 257 if (data.imported_json) { ··· 451 299 if (data.has_summary) { 452 300 const summaryTab = document.getElementById('summaryTab'); 453 301 summaryTab.style.display = 'block'; 454 - } 455 - 456 - // Show re-run section if import is completed or failed and file exists 457 - if (data.import_json && data.import_json.file_path) { 458 - // Show re-run section for completed or failed imports 459 - if (data.imported_json || (data.import_json.task_id && !data.imported_json)) { 460 - const rerunSection = document.getElementById('rerunSection'); 461 - rerunSection.style.display = 'block'; 462 - 463 - // Load facets for the dropdown 464 - loadFacets(); 465 - } 466 302 } 467 303 }) 468 304 .catch(err => {

-92

apps/import/routes.py

··· 1 1 from __future__ import annotations 2 2 3 - import datetime 4 3 import time 5 4 from pathlib import Path 6 5 from typing import Any ··· 12 11 from think.callosum import callosum_send 13 12 from think.detect_created import detect_created 14 13 from think.importer_utils import ( 15 - archive_imported_results, 16 14 build_import_info, 17 15 get_import_details, 18 16 list_import_timestamps, ··· 403 401 return jsonify({"error": "Failed to submit task"}), 500 404 402 405 403 return jsonify({"status": "ok", "task_id": task_id}) 406 - 407 - 408 - @import_bp.route("/api/<timestamp>/rerun", methods=["POST"]) 409 - def import_rerun(timestamp: str) -> Any: 410 - """Re-run an import with optionally updated facet.""" 411 - journal_root = Path(state.journal_root) 412 - 413 - # Check if import exists 414 - import_dir = journal_root / "imports" / timestamp 415 - if not import_dir.exists(): 416 - return jsonify({"error": "Import not found"}), 404 417 - 418 - # Read import metadata using utility function 419 - try: 420 - metadata = read_import_metadata(journal_root=journal_root, timestamp=timestamp) 421 - except FileNotFoundError: 422 - return jsonify({"error": "Import metadata not found"}), 404 423 - except Exception as e: 424 - return jsonify({"error": f"Failed to read import metadata: {str(e)}"}), 500 425 - 426 - # Get file path from metadata 427 - file_path = metadata.get("file_path") 428 - if not file_path: 429 - return jsonify({"error": "File path not found in metadata"}), 500 430 - 431 - # Check if file still exists 432 - if not Path(file_path).exists(): 433 - return jsonify({"error": "Original file no longer exists"}), 404 434 - 435 - # Get new facet/setting from request 436 - data = request.get_json(force=True) 437 - new_facet = data.get("facet", "").strip() or None 438 - new_setting = data.get("setting", "").strip() or None 439 - 440 - # Check if values changed 441 - facet_changed = new_facet != metadata.get("facet") 442 - setting_changed = new_setting != metadata.get("setting") 443 - 444 - # Update metadata with new values and rerun timestamp 445 - if facet_changed or setting_changed or "setting" not in metadata: 446 - updates = { 447 - "facet": new_facet, 448 - "setting": new_setting, 449 - "rerun_at": time.time() * 1000, 450 - "rerun_datetime": datetime.datetime.now().isoformat(), 451 - } 452 - try: 453 - update_import_metadata_fields( 454 - journal_root=journal_root, 455 - timestamp=timestamp, 456 - updates=updates, 457 - ) 458 - except Exception as e: 459 - return jsonify({"error": f"Failed to update metadata: {str(e)}"}), 500 460 - 461 - # Archive previous processing results using utility function 462 - archive_imported_results(journal_root=journal_root, timestamp=timestamp) 463 - 464 - # Generate task ID 465 - task_id = str(int(time.time() * 1000)) 466 - 467 - # Build command 468 - cmd = ["think-importer", file_path, timestamp] 469 - if new_facet: 470 - cmd.extend(["--facet", new_facet]) 471 - if new_setting: 472 - cmd.extend(["--setting", new_setting]) 473 - 474 - # Store task_id in metadata 475 - try: 476 - update_import_metadata_fields( 477 - journal_root=journal_root, 478 - timestamp=timestamp, 479 - updates={"task_id": task_id}, 480 - ) 481 - except Exception as e: 482 - return jsonify({"error": f"Failed to update metadata: {str(e)}"}), 500 483 - 484 - # Emit task request to Callosum 485 - if not callosum_send("supervisor", "request", ref=task_id, cmd=cmd): 486 - return jsonify({"error": "Failed to submit task"}), 500 487 - 488 - return jsonify( 489 - { 490 - "status": "ok", 491 - "facet": new_facet, 492 - "setting": new_setting, 493 - "task_id": task_id, 494 - } 495 - )

-33

tests/test_importer_utils.py

··· 7 7 import pytest 8 8 9 9 from think.importer_utils import ( 10 - archive_imported_results, 11 10 build_import_info, 12 11 calculate_duration_from_files, 13 12 get_import_details, ··· 320 319 """Test getting details for non-existent import.""" 321 320 with pytest.raises(FileNotFoundError): 322 321 get_import_details(temp_journal, "20250101_999999") 323 - 324 - 325 - def test_archive_imported_results(temp_journal): 326 - """Test archiving imported.json before re-run.""" 327 - timestamp = "20250101_210000" 328 - import_dir = temp_journal / "imports" / timestamp 329 - import_dir.mkdir(parents=True) 330 - 331 - # Create imported.json 332 - imported_path = import_dir / "imported.json" 333 - imported_path.write_text('{"test": "data"}', encoding="utf-8") 334 - 335 - # Archive it 336 - archive_imported_results(temp_journal, timestamp) 337 - 338 - # Original should be gone 339 - assert not imported_path.exists() 340 - 341 - # Archive should exist 342 - archives = list(import_dir.glob("imported.*.json.bak")) 343 - assert len(archives) == 1 344 - assert archives[0].read_text(encoding="utf-8") == '{"test": "data"}' 345 - 346 - 347 - def test_archive_imported_results_no_file(temp_journal): 348 - """Test archiving when imported.json doesn't exist (should be no-op).""" 349 - timestamp = "20250101_220000" 350 - import_dir = temp_journal / "imports" / timestamp 351 - import_dir.mkdir(parents=True) 352 - 353 - # Should not raise error 354 - archive_imported_results(temp_journal, timestamp)

+1 -31

think/importer_utils.py

··· 1 1 """Utility functions for import operations. 2 2 3 3 This module contains reusable logic for managing imports in the journal, 4 - extracted from convey/views/import.py to be usable in CLI tools and other contexts. 4 + extracted from apps/import/routes.py to be usable in CLI tools and other contexts. 5 5 """ 6 6 7 7 from __future__ import annotations 8 8 9 9 import json 10 - import time 11 10 from pathlib import Path 12 11 13 12 # ============================================================================ ··· 465 464 result["has_summary"] = True 466 465 467 466 return result 468 - 469 - 470 - # ============================================================================ 471 - # Re-run Support 472 - # ============================================================================ 473 - 474 - 475 - def archive_imported_results( 476 - journal_root: Path, 477 - timestamp: str, 478 - ) -> None: 479 - """Archive imported.json to timestamped backup before re-run. 480 - 481 - Args: 482 - journal_root: Root journal directory 483 - timestamp: Import timestamp 484 - """ 485 - import_dir = journal_root / "imports" / timestamp 486 - imported_json_path = import_dir / "imported.json" 487 - 488 - if not imported_json_path.exists(): 489 - return 490 - 491 - try: 492 - # Archive the old results 493 - archive_path = import_dir / f"imported.{int(time.time())}.json.bak" 494 - imported_json_path.rename(archive_path) 495 - except Exception: 496 - pass # Continue even if archiving fails

Configure Feed

Configure Feed