CLI utility to ingest embedded json metadata from yt-dlp downloads to a SQLite database file
yt-dlp
feat: Don't re-index videos that are already found in database
+43
-3
+20
.sqlx/query-2a400bc9d31aa3bb6ca352a2985d8965ba3b11ede82a0f9d868df985a4a36724.json
+20
.sqlx/query-2a400bc9d31aa3bb6ca352a2985d8965ba3b11ede82a0f9d868df985a4a36724.json
···
1
+
{
2
+
"db_name": "SQLite",
3
+
"query": "SELECT video_path FROM videos;",
4
+
"describe": {
5
+
"columns": [
6
+
{
7
+
"name": "video_path",
8
+
"ordinal": 0,
9
+
"type_info": "Text"
10
+
}
11
+
],
12
+
"parameters": {
13
+
"Right": 0
14
+
},
15
+
"nullable": [
16
+
false
17
+
]
18
+
},
19
+
"hash": "2a400bc9d31aa3bb6ca352a2985d8965ba3b11ede82a0f9d868df985a4a36724"
20
+
}
+23
-3
src/lib.rs
+23
-3
src/lib.rs
···
1
1
use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
2
2
use matroska::MatroskaError;
3
3
use sqlx::{Pool, Sqlite};
4
-
use std::path::PathBuf;
4
+
use std::{path::PathBuf, str::FromStr};
5
5
use thiserror::Error;
6
6
use walkdir::WalkDir;
7
7
···
77
77
.collect()
78
78
});
79
79
80
+
log::info!("Found {} videos to index.", files.len());
81
+
80
82
if files.is_empty() {
81
83
return Err(IndexError::NoVideos);
82
84
}
83
-
log::info!("Found {} videos to index.", files.len());
85
+
86
+
// Get Vec<PathBuf> of all videos already indexed
87
+
let videos_indexed: Vec<PathBuf> = sqlx::query!("SELECT video_path FROM videos;")
88
+
.fetch_all(db_pool)
89
+
.await
90
+
.unwrap()
91
+
.iter()
92
+
.map(|i| PathBuf::from_str(&i.video_path).unwrap())
93
+
.collect();
94
+
95
+
log::info!("DB has {} videos", videos_indexed.len());
96
+
97
+
// Filter files form files already found in videos_indexed
98
+
let files: Vec<&PathBuf> = files
99
+
.iter()
100
+
.filter(|x| !videos_indexed.contains(x))
101
+
.collect();
102
+
103
+
log::info!("File-list reduced to {} by comparing to DB.", files.len());
84
104
85
105
let bar = multi_progress.add(ProgressBar::new(files.len().try_into().unwrap()));
86
106
···
94
114
}
95
115
96
116
for path in files {
97
-
if let Err(error) = index_video(&path, db_pool).await {
117
+
if let Err(error) = index_video(path, db_pool).await {
98
118
let path_str = path.to_string_lossy().to_string();
99
119
let err_str = error.to_string();
100
120