CLI utility to ingest embedded json metadata from yt-dlp downloads to a SQLite database file
yt-dlp
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat: Don't re-index videos that are already found in database

0xBA5E64 200b24bd 74e97431

+43 -3
+20
.sqlx/query-2a400bc9d31aa3bb6ca352a2985d8965ba3b11ede82a0f9d868df985a4a36724.json
··· 1 + { 2 + "db_name": "SQLite", 3 + "query": "SELECT video_path FROM videos;", 4 + "describe": { 5 + "columns": [ 6 + { 7 + "name": "video_path", 8 + "ordinal": 0, 9 + "type_info": "Text" 10 + } 11 + ], 12 + "parameters": { 13 + "Right": 0 14 + }, 15 + "nullable": [ 16 + false 17 + ] 18 + }, 19 + "hash": "2a400bc9d31aa3bb6ca352a2985d8965ba3b11ede82a0f9d868df985a4a36724" 20 + }
+23 -3
src/lib.rs
··· 1 1 use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; 2 2 use matroska::MatroskaError; 3 3 use sqlx::{Pool, Sqlite}; 4 - use std::path::PathBuf; 4 + use std::{path::PathBuf, str::FromStr}; 5 5 use thiserror::Error; 6 6 use walkdir::WalkDir; 7 7 ··· 77 77 .collect() 78 78 }); 79 79 80 + log::info!("Found {} videos to index.", files.len()); 81 + 80 82 if files.is_empty() { 81 83 return Err(IndexError::NoVideos); 82 84 } 83 - log::info!("Found {} videos to index.", files.len()); 85 + 86 + // Get Vec<PathBuf> of all videos already indexed 87 + let videos_indexed: Vec<PathBuf> = sqlx::query!("SELECT video_path FROM videos;") 88 + .fetch_all(db_pool) 89 + .await 90 + .unwrap() 91 + .iter() 92 + .map(|i| PathBuf::from_str(&i.video_path).unwrap()) 93 + .collect(); 94 + 95 + log::info!("DB has {} videos", videos_indexed.len()); 96 + 97 + // Filter files form files already found in videos_indexed 98 + let files: Vec<&PathBuf> = files 99 + .iter() 100 + .filter(|x| !videos_indexed.contains(x)) 101 + .collect(); 102 + 103 + log::info!("File-list reduced to {} by comparing to DB.", files.len()); 84 104 85 105 let bar = multi_progress.add(ProgressBar::new(files.len().try_into().unwrap())); 86 106 ··· 94 114 } 95 115 96 116 for path in files { 97 - if let Err(error) = index_video(&path, db_pool).await { 117 + if let Err(error) = index_video(path, db_pool).await { 98 118 let path_str = path.to_string_lossy().to_string(); 99 119 let err_str = error.to_string(); 100 120