CLI utility to ingest embedded json metadata from yt-dlp downloads to a SQLite database file
yt-dlp
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

refactor: Enable remove_missing on index, and refactor general args & imports

0xBA5E64 0e6ffd08 edf30df8

+49 -22
+29 -11
src/lib.rs
··· 58 58 pub async fn index_videos_recursively( 59 59 in_dir: PathBuf, 60 60 db_pool: &Pool<Sqlite>, 61 - headless_mode: bool, 61 + remove_missing: bool, 62 + headless: bool, 62 63 multi_progress: MultiProgress, 63 64 ) -> Result<(), IndexError> { 64 65 let files: Vec<PathBuf> = tokio::task::block_in_place(|| { ··· 92 93 .map(|i| PathBuf::from_str(&i.video_path).unwrap()) 93 94 .collect(); 94 95 96 + if remove_missing { 97 + for file in &videos_indexed { 98 + let path_str = file.to_string_lossy().to_string(); 99 + let file_exists = file.try_exists().unwrap_or(false); 100 + 101 + if !file_exists { 102 + log::error!("Couldn't find \"{}\" - Removing from Database", path_str); 103 + sqlx::query!("DELETE FROM failed_videos WHERE video_path = ?1", path_str) 104 + .execute(db_pool) 105 + .await 106 + .map_err(|_| IndexError::DatabaseError)?; 107 + continue; 108 + } 109 + } 110 + } 111 + 95 112 log::info!("DB has {} videos", videos_indexed.len()); 96 113 97 114 // Filter files form files already found in videos_indexed ··· 104 121 105 122 let bar = multi_progress.add(ProgressBar::new(files.len().try_into().unwrap())); 106 123 107 - if !headless_mode { 124 + if !headless { 108 125 bar.set_style( 109 126 ProgressStyle::with_template( 110 127 "[{elapsed_precise} / {duration_precise}] {wide_bar} [{human_pos}/{human_len}]", ··· 114 131 } 115 132 116 133 for path in files { 134 + let path_str = path.to_string_lossy().to_string(); 135 + 117 136 if let Err(error) = index_video(path, db_pool).await { 118 - let path_str = path.to_string_lossy().to_string(); 119 137 let err_str = error.to_string(); 120 138 121 139 log::error!("Couldn't index \"{}\" - {}", path_str, err_str); ··· 132 150 continue; 133 151 }; 134 152 135 - if !headless_mode { 153 + if !headless { 136 154 bar.inc(1); 137 155 } 138 156 } 139 157 140 - if !headless_mode { 158 + if !headless { 141 159 bar.finish(); 142 160 } 143 161 Ok(()) ··· 145 163 146 164 pub async fn reindex_failed_videos( 147 165 db_pool: &Pool<Sqlite>, 148 - removed_failed: bool, 149 - headless_mode: bool, 166 + remove_missing: bool, 167 + headless: bool, 150 168 multi_progress: MultiProgress, 151 169 ) -> Result<(), IndexError> { 152 170 // Get Vec<PathBuf> of all videos already indexed ··· 166 184 167 185 let bar = multi_progress.add(ProgressBar::new(previous_failed.len().try_into().unwrap())); 168 186 169 - if !headless_mode { 187 + if !headless { 170 188 bar.set_style( 171 189 ProgressStyle::with_template( 172 190 "[{elapsed_precise} / {duration_precise}] {wide_bar} [{human_pos}/{human_len}]", ··· 178 196 let path_str = path.to_string_lossy().to_string(); 179 197 let file_exists = path.try_exists().unwrap_or(false); 180 198 181 - if removed_failed && !file_exists { 199 + if remove_missing && !file_exists { 182 200 log::error!("Couldn't find \"{}\" - Removing from Database", path_str); 183 201 sqlx::query!("DELETE FROM failed_videos WHERE video_path = ?1", path_str) 184 202 .execute(db_pool) ··· 203 221 continue; 204 222 }; 205 223 206 - if !headless_mode { 224 + if !headless { 207 225 bar.inc(1); 208 226 } 209 227 } 210 228 211 - if !headless_mode { 229 + if !headless { 212 230 bar.finish(); 213 231 } 214 232
+20 -11
src/main.rs
··· 1 1 use clap::{Parser, Subcommand}; 2 2 use sqlx::migrate; 3 - use sqlx::sqlite::{SqliteAutoVacuum, SqliteConnectOptions, SqlitePoolOptions}; 3 + use sqlx::sqlite::*; 4 4 use std::path::PathBuf; 5 - use ydjr::{index_videos_recursively, reindex_failed_videos}; 5 + use ydjr::*; 6 6 7 7 #[derive(Parser)] 8 8 #[command(version, about, long_about = None)] 9 - struct Args { 9 + struct YdjrArgs { 10 10 /// Where to write or open the database file from 11 11 #[arg(long, default_value = "./db.sqlite")] 12 12 db: PathBuf, 13 13 14 + /// Remove videos from db if no longer found on filesystem 15 + #[arg(long, short, default_value = "false")] 16 + remove_missing: bool, 17 + 14 18 /// Headless mode, fitting if invoked automatically 15 19 #[arg(long, short = 'H')] 16 20 headless: bool, ··· 27 31 IndexDirectory { path: PathBuf }, 28 32 /// Retry failed indexings found in the database 29 33 #[command(name = "retry-failed")] 30 - ReIndexFailed { 31 - #[arg(long, short, default_value = "false")] 32 - remove_failed: bool, 33 - }, 34 + ReIndexFailed, 34 35 } 35 36 36 37 #[tokio::main] ··· 45 46 46 47 logger_wrapper.try_init()?; 47 48 48 - let args = Args::parse(); 49 + let args = YdjrArgs::parse(); 49 50 50 51 let db_pool = SqlitePoolOptions::new() 51 52 .max_connections(32) ··· 65 66 66 67 match args.cmd { 67 68 CmdOption::IndexDirectory { path } => { 68 - index_videos_recursively(path, &db_pool, args.headless, multi_progress).await? 69 + index_videos_recursively( 70 + path, 71 + &db_pool, 72 + args.remove_missing, 73 + args.headless, 74 + multi_progress, 75 + ) 76 + .await? 69 77 } 70 - CmdOption::ReIndexFailed { remove_failed } => { 71 - reindex_failed_videos(&db_pool, remove_failed, args.headless, multi_progress).await? 78 + CmdOption::ReIndexFailed => { 79 + reindex_failed_videos(&db_pool, args.remove_missing, args.headless, multi_progress) 80 + .await? 72 81 } 73 82 } 74 83 Ok(())