···11-# `tracksync`: if `rsync` was ID3-aware
11+# `tunz`: if `rsync` was ID3-aware
2233-`tracksync` synchronizes music files from a source to a destination, keeping a database for both.
33+`tunz` synchronizes music files from a source to a destination, keeping a database for both.
4455Tracks need to be added to the source database first, and then can be synced on the destination.
66···11111212Pass `-h` to each subcommand to understand how to use it!
13131414-`tracksync` can also create hardlinks instead of copies of your files: pass the `--link` flag to `sync` to do so.
1414+`tunz` can also create hardlinks instead of copies of your files: pass the `--link` flag to `sync` to do so.
15151616## Installing
17171818```sh
1919-cargo install tracksync
1919+cargo install tunz
2020```
21212222### From sources
···2626Once you have that setup:
27272828```sh
2929-git clone https://github.com/gsora/tracksync
3030-cd tracksync
2929+git clone https://github.com/gsora/tunz
3030+cd tunz
3131cargo build --release
3232-./target/release/tracksync -h
3232+./target/release/tunz -h
3333```
34343535## Filtering
···104104105105The database schema might break suddenly, making your source and destination(s) libraries unusable: a `rescan` command
106106is in the works -- I will make sure to keep those at a minimum.
107107+108108+## conversion
109109+110110+Debian deps:
111111+- libavutil-dev
112112+- libavformat-dev
113113+- libavfilter-dev
114114+- libavdevice-dev
···11-use clap::{command, Parser, Subcommand};
11+use clap::{Parser, Subcommand};
2233use crate::cmd;
44···2727 /// Cleans destination of uncleanly-copied files.
2828 Clean(cmd::clean::Args),
29293030+ /// Settings allows modifying transcoding and filtering settings.
3131+ Settings {
3232+ #[command(subcommand)]
3333+ command: Settings,
3434+ },
3535+}
3636+3737+#[derive(Subcommand)]
3838+pub enum Settings {
3039 /// Filter tracks to copy over to a destination.
3140 Filter(cmd::filter::Args),
4141+4242+ /// Convert files from one format to another during sync.
4343+ Transcode(cmd::transcode::Args),
3244}
+16-53
src/cmd/add.rs
···55use anyhow::{Context, Result};
66use clap::Args as ClapArgs;
77use futures::{executor::block_on, future::try_join_all};
88-use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
98use model::FileState;
1091110#[derive(ClapArgs, Debug)]
1211pub struct Args {
1313- /// Directory in which tracksync will store its local database.
1212+ /// Directory in which tunz will store its local database.
1413 #[arg(short, long, default_value_t = db::default_database_dir().to_str().unwrap().to_owned())]
1514 pub database_path: String,
16151717- /// A path in which tracksync will look for music files.
1616+ /// A path in which tunz will look for music files.
1817 /// Specify more than one for multiple sources.
1918 #[arg(short, long = "source", value_name = "SOURCE", action = clap::ArgAction::Append)]
2019 pub sources: Option<Vec<String>>,
21202121+ // TODO: we could understand if the database path is destination or not by detecting
2222+ // if it's a mount point and not the local disk.
2223 /// Specifies if the database is to be written is a destination one.
2324 #[arg(
2425 long = "destination",
···4142}
42434344pub async fn run(args: Args, update: bool) -> Result<()> {
4444- let val_res = args.validate();
4545-4645 match update {
4746 true => {}
4848- false => val_res?,
4747+ false => args.validate()?,
4948 };
50495150 log::debug!("CLI args: {:?}", args);
···6362 .with_context(|| "Cannot fetch track directories from database")?,
6463 };
65646666- let mp = MultiProgress::new();
6765 let mut tracks = vec![];
68666967 for source in &sources {
···8280 sources
8381 .into_iter()
8482 .map(|source| {
8585- traverse_and_add_param(&db, &mp, source, {
8383+ traverse_and_add_param(&db, source, {
8684 let tracks_set = tracks_set.clone();
87858888- move |path, db, pb| match update {
8989- false => add_dupe_checker(path, db, pb),
8686+ move |path, db| match update {
8787+ false => add_dupe_checker(path, db),
9088 true => Ok(!tracks_set.contains(path)),
9189 }
9290 })
···10199 0 => log::info!("Imported {} tracks", totals.0),
102100 _ => match update {
103101 false => log::info!(
104104- "Imported {} new tracks, but found {} duplicates",
102102+ "Imported {} new tracks, found {} duplicates",
105103 totals.0,
106104 totals.1
107105 ),
···110108 };
111109112110 if update {
113113- let prog = mp.add(
114114- ProgressBar::new_spinner()
115115- .with_message("Looking for tracks not on disk anymore...")
116116- .with_style(ProgressStyle::default_spinner()),
117117- );
118118-119119- prog.enable_steady_tick(std::time::Duration::from_millis(50));
120120-121111 // look for files in db that are not on the filesystem anymore
122112 let track_iter = db
123113 .tracks_iter()
···132122 match tp.exists() {
133123 true => {}
134124 false => {
135135- prog.set_message(format!(
136136- "Found track in database not existing on filesystem, deleting: {}",
137137- track.file_path,
138138- ));
139139-140125 db.delete(track.id)
141126 .await
142127 .with_context(|| "Cannot delete track from database.")?;
143128 }
144129 }
145130 }
146146-147147- prog.finish();
148148- mp.remove(&prog);
149131 }
150132151133 Ok(())
152134}
153135154154-fn add_dupe_checker(path: &String, db: &db::Instance, pb: &indicatif::ProgressBar) -> Result<bool> {
136136+fn add_dupe_checker(path: &String, db: &db::Instance) -> Result<bool> {
155137 block_on(async {
156138 if db.exists(path.clone()).await? {
157157- pb.set_message(format!("Found duplicate at {}", path.clone()));
158139 return Ok(true);
159140 }
160141···164145165146pub(crate) async fn traverse_and_add_param<F>(
166147 db: &db::Instance,
167167- mp: &MultiProgress,
168148 path: String,
169149 dupe_checker: F,
170150) -> Result<(u64, u64)>
171151where
172172- F: FnOnce(&String, &db::Instance, &indicatif::ProgressBar) -> Result<bool> + Clone,
152152+ F: FnOnce(&String, &db::Instance) -> Result<bool> + Clone,
173153{
174174- let paths = fs::traverse(&path).await;
175175-176176- let base_msg = format!("Reading {}...", path.clone());
177177-178178- let prog = mp.add(
179179- ProgressBar::new_spinner()
180180- .with_message(base_msg.clone())
181181- .with_style(ProgressStyle::default_spinner()),
182182- );
154154+ log::info!("scanning {}", path);
183155184184- prog.enable_steady_tick(std::time::Duration::from_millis(50));
156156+ let paths = fs::traverse(&path).await;
185157186158 let mut new_tracks = 0;
187159 let mut duplicate = 0;
···190162 let p = p?.clone();
191163192164 let dc = dupe_checker.clone();
193193- if dc(&p, db, &prog)? {
165165+ if dc(&p, db)? {
194166 duplicate += 1;
195167 continue;
196168 }
···202174 let mut track: model::Track = model::RawTrack { tags, path: p }.into();
203175 track.file_state = FileState::Copied;
204176177177+ log::info!("new track: {}", track);
178178+205179 db.insert_track(&track)
206180 .await
207181 .with_context(|| format!("Cannot write track data to database"))?;
208182209209- prog.set_message(format!(
210210- "{}\nFound track: {} - {}, from {}",
211211- base_msg.clone(),
212212- track.title,
213213- track.artist,
214214- track.album
215215- ));
216216-217183 new_tracks += 1;
218184 }
219219-220220- prog.finish();
221221- mp.remove(&prog);
222185223186 db.insert_directory(path).await?;
224187
+1-1
src/cmd/dupes.rs
···5566#[derive(ClapArgs, Debug)]
77pub struct Args {
88- /// Directory in which tracksync will store its local database.
88+ /// Directory in which tunz will store its local database.
99 #[arg(short, long, default_value_t = db::default_database_dir().to_str().unwrap().to_owned())]
1010 pub database_path: String,
1111}
+1-1
src/cmd/filter.rs
···6677#[derive(ClapArgs)]
88pub struct Args {
99- /// Path where tracksync database is stored.
99+ /// Path where tunz database is stored.
1010 #[arg(long)]
1111 pub destination: Option<String>,
1212
+1
src/cmd/mod.rs
···44pub mod error;
55pub mod filter;
66pub mod sync;
77+pub mod transcode;
+92-122
src/cmd/sync.rs
···11use crate::cmd::*;
22use crate::db;
33+use crate::ffmpeg;
34use crate::model;
44-use anyhow::anyhow;
55use anyhow::Ok;
66+use anyhow::anyhow;
67use anyhow::{Context, Result};
78use clap::Args as ClapArgs;
88-use fs_extra::file::{copy_with_progress, CopyOptions};
99-use indicatif::MultiProgress;
99+use fs_extra::file::{CopyOptions, copy_with_progress};
1010+use futures::StreamExt;
1111+use futures::TryStreamExt;
1212+use futures::stream;
1013use std::collections::hash_set;
1111-use std::os::unix::fs::MetadataExt;
1414+use std::str::FromStr;
12151316#[derive(ClapArgs)]
1417pub struct Args {
1515- /// Path where to look for tracksync source data.
1818+ /// Path where to look for tunz source data.
1619 #[arg(short, long, default_value_t = db::default_database_dir().to_str().unwrap().to_owned())]
1720 pub database_path: String,
18211919- /// Path where to store tracksync's database, as well as music files.
2222+ /// Path where to store tunz's database, as well as music files.
2023 #[arg(long)]
2124 pub destination: Option<String>,
2225···3235 /// Instead of copying the files over to the specified destination, create an hardlink.
3336 #[arg(long, default_value_t = false)]
3437 pub link: bool,
3838+3939+ /// Amount of threads to use for copy and transcoding, defaults to number of CPUs
4040+ /// available.
4141+ #[arg(long)]
4242+ pub threads: Option<usize>,
3543}
36443745impl Args {
···48564957pub async fn run(args: Args) -> Result<()> {
5058 args.validate()?;
5959+6060+ let threads = match args.threads {
6161+ Some(t) => t,
6262+ None => num_cpus::get(),
6363+ };
51645265 let dest_dir = args.destination.unwrap();
5366···95108 if !args.no_delete {
96109 run_delete(&dest_db, &dest_dir, reverse_diff, filters.as_ref()).await?
97110 }
111111+112112+ log::info!("using {} threads", threads);
9811399114 run_copy(
100115 &local_db,
···103118 diff,
104119 filters.as_ref(),
105120 args.link,
121121+ threads,
106122 )
107123 .await?;
108124109125 Ok(())
110126}
111127112112-fn progress_bar(size: u64, style: indicatif::ProgressStyle) -> indicatif::ProgressBar {
113113- let bar = indicatif::ProgressBar::new(size);
114114- bar.set_style(style);
115115-116116- bar
117117-}
118118-119119-fn total_style() -> indicatif::ProgressStyle {
120120- indicatif::ProgressStyle::with_template(
121121- "Total progress:\n[{percent}% {wide_bar:.green}] {human_pos}/{human_len} {elapsed}\n\n",
122122- )
123123- .unwrap()
124124- .progress_chars("##-")
125125-}
126126-127127-fn delete_style() -> indicatif::ProgressStyle {
128128- indicatif::ProgressStyle::with_template(
129129- "Deleting old tracks:\n[{percent}% {wide_bar:.green}] {human_pos}/{human_len} {elapsed}\n\n",
130130- )
131131- .unwrap()
132132- .progress_chars("##-")
133133-}
134134-135135-fn track_style() -> indicatif::ProgressStyle {
136136- indicatif::ProgressStyle::with_template(
137137- "{msg}\n[{percent}% {wide_bar:.green}] {decimal_bytes:>7}/{decimal_total_bytes:7} {elapsed}\n\n",
138138- )
139139- .unwrap()
140140- .progress_chars("##-")
141141-}
142142-143128async fn diff_databases(
144129 source: &db::Instance,
145130 destination: &db::Instance,
···228213 diff: Vec<String>,
229214 filters: Option<&Vec<crate::filter::ScriptRuntime>>,
230215 link: bool,
216216+ threads: usize,
231217) -> Result<()> {
232232- let diff_len = diff.len();
218218+ let tr_settings = dest_db.get_transcoding_settings().await?;
233219234220 let tracks = filter_tracks(
235221 local_db
···240226 false,
241227 )?;
242228243243- // Copy tracks
244244- let mp = MultiProgress::new();
229229+ stream::iter(tracks)
230230+ .map(|track| {
231231+ let trs = tr_settings.clone();
232232+ async move { do_copy(track, dest_db, dest_dir, trs, link).await }
233233+ })
234234+ .buffer_unordered(threads)
235235+ .try_for_each(|_| async move { Ok(()) })
236236+ .await
237237+}
245238246246- let total_bar = mp.add(progress_bar(diff_len as u64, total_style()));
247247-248248- total_bar.tick();
249249-250250- for track in tracks {
251251- copy(track, &dest_db, &dest_dir, &mp, link).await?;
252252- total_bar.inc(1);
239239+async fn do_copy(
240240+ track: model::Track,
241241+ dest_db: &db::Instance,
242242+ dest_dir: &String,
243243+ transcoding_settings: Option<ffmpeg::Quality>,
244244+ link: bool,
245245+) -> Result<()> {
246246+ if let Some(ts) = transcoding_settings.clone() {
247247+ log::info!("copying and transcoding {} ({:?})", track, ts);
248248+ } else {
249249+ log::info!("copying {}", track);
253250 }
254254-255255- total_bar.finish();
256256-257257- Ok(())
251251+ copy(track, &dest_db, &dest_dir, transcoding_settings, link).await
258252}
259253260254async fn dry_run_copy(
···326320 true,
327321 )?;
328322329329- let mp = MultiProgress::new();
330330-331331- let total_bar = mp.add(progress_bar(diff_len as u64, delete_style()));
332332-333333- total_bar.tick();
334334-335323 for track in tracks {
336336- delete(track, &dest_db, &dest_dir, &mp).await?;
337337- total_bar.inc(1);
324324+ log::info!("deleting {}", track);
325325+ delete(track, &dest_db, &dest_dir).await?;
338326 }
339327340340- total_bar.finish();
341341-342328 Ok(())
343329}
344330345345-async fn delete(
346346- track: model::Track,
347347- dest_db: &db::Instance,
348348- dest_dir: &String,
349349- mp: &indicatif::MultiProgress,
350350-) -> Result<()> {
331331+async fn delete(track: model::Track, dest_db: &db::Instance, dest_dir: &String) -> Result<()> {
351332 let track_storage_path = track.storage_path(&dest_dir);
352333353353- let bar = mp.add(
354354- progress_bar(1, track_style()).with_message(format!("Deleting: {}", track_storage_path)),
355355- );
356356-357334 dest_db.delete(track.id).await?;
358335 std::fs::remove_file(track_storage_path.clone())
359336 .with_context(|| format!("Cannot delete file {}", track_storage_path.clone()))?;
360360-361361- bar.inc(1);
362362-363363- mp.remove(&bar);
364364-365337 Ok(())
366338}
367339···369341 track: model::Track,
370342 dest_db: &db::Instance,
371343 dest_dir: &String,
372372- mp: &indicatif::MultiProgress,
344344+ transcoding_settings: Option<ffmpeg::Quality>,
373345 link: bool,
374346) -> Result<()> {
375347 let track_storage_path = track.storage_path(&dest_dir);
···395367 )
396368 })?;
397369398398- let orig_file_path = std::path::Path::new(&track.file_path);
399399- let orig_file = std::fs::File::open(orig_file_path).with_context(|| {
400400- format!(
401401- "Cannot open origin file {}",
402402- orig_file_path.to_str().unwrap()
403403- )
404404- })?;
370370+ let opts = CopyOptions::new().overwrite(true);
405371406406- let orig_file_meta = orig_file.metadata().with_context(|| {
407407- format!(
408408- "Cannot obtain metadata information of {}",
409409- orig_file_path.to_str().unwrap()
410410- )
411411- })?;
372372+ let track_for_thread = track.clone();
373373+ let dest_dir_for_thread = dest_dir.clone();
374374+ let ts_for_thread = transcoding_settings.clone();
412375413413- let bar = mp.add(
414414- progress_bar(orig_file_meta.size(), track_style()).with_message(format!(
415415- "Copying: {}\nTo: {}",
416416- track.file_path, track_storage_path
417417- )),
418418- );
376376+ async_std::task::spawn_blocking(move || {
377377+ if let Some(ts) = ts_for_thread {
378378+ ffmpeg::convert(
379379+ &track_for_thread,
380380+ std::path::PathBuf::from_str(
381381+ &track_for_thread
382382+ .storage_path_with_extension(&dest_dir_for_thread, &ts.codec.extension()),
383383+ )
384384+ .unwrap(),
385385+ &ts,
386386+ )?;
387387+ } else {
388388+ if link {
389389+ std::fs::hard_link(
390390+ track_for_thread.file_path.clone(),
391391+ track_storage_path.clone(),
392392+ )?;
393393+ } else {
394394+ match copy_with_progress(
395395+ track_for_thread.file_path.clone(),
396396+ track_storage_path.clone(),
397397+ &opts,
398398+ |_| {},
399399+ ) {
400400+ std::result::Result::Ok(_) => {}
401401+ Err(err) => {
402402+ return Err(error::Error::CopyError(err)).with_context(|| {
403403+ format!(
404404+ "Cannot copy {} to {}",
405405+ track_for_thread.file_path, track_storage_path
406406+ )
407407+ });
408408+ }
409409+ };
410410+ }
411411+ }
419412420420- let opts = CopyOptions::new().overwrite(true);
421421-422422- if link {
423423- std::fs::hard_link(track.file_path.clone(), track_storage_path.clone())?;
424424- } else {
425425- match copy_with_progress(
426426- track.file_path.clone(),
427427- track_storage_path.clone(),
428428- &opts,
429429- |ph| {
430430- bar.set_position(ph.copied_bytes);
431431- },
432432- ) {
433433- std::result::Result::Ok(_) => {}
434434- Err(err) => {
435435- return Err(error::Error::CopyError(err)).with_context(|| {
436436- format!("Cannot copy {} to {}", track.file_path, track_storage_path)
437437- });
438438- }
439439- };
440440- // .with_context(|| format!("Cannot copy {} to {}", track.file_path, track_storage_path))?;
441441- }
413413+ Ok(())
414414+ })
415415+ .await?;
442416443417 // step 3: update the destination track with the new state
444418 dest_track.file_state = crate::model::FileState::Copied;
···446420 .insert_track(&dest_track)
447421 .await
448422 .with_context(|| "Cannot insert copy finished track in destination database")?;
449449-450450- bar.finish();
451451-452452- mp.remove(&bar);
453423454424 Ok(())
455425}
+76
src/cmd/transcode.rs
···11+use crate::{
22+ db,
33+ ffmpeg::{Bitrate, BitrateParser, Codec, SamplingRate, SamplingRateParser},
44+};
55+use anyhow::{Result, anyhow};
66+use clap::Args as ClapArgs;
77+88+#[derive(ClapArgs, Debug)]
99+pub struct Args {
1010+ /// Path where tunz database is stored.
1111+ #[arg(long)]
1212+ pub destination: String,
1313+1414+ /// Enable or disable transcoding on copy.
1515+ #[arg(long)]
1616+ pub enabled: Option<bool>,
1717+1818+ /// Codec to transcode files to.
1919+ #[arg(long)]
2020+ pub codec: Option<Codec>,
2121+2222+ /// Bitrate to transcode at.
2323+ #[arg(long, value_parser = BitrateParser{})]
2424+ pub bitrate: Option<Bitrate>,
2525+2626+ /// Sample rate to transcode at.
2727+ #[arg(long, value_parser = SamplingRateParser{})]
2828+ pub sampling_rate: Option<SamplingRate>,
2929+}
3030+3131+impl Args {
3232+ pub fn validate(&self) -> Result<()> {
3333+ match self.enabled {
3434+ Some(s) => match s {
3535+ true => {
3636+ if !(self.bitrate.is_some()
3737+ || self.codec.is_some()
3838+ || self.sampling_rate.is_some())
3939+ {
4040+ return Err(anyhow!("can't enable transcoding with no settings!"));
4141+ }
4242+4343+ Ok(())
4444+ }
4545+ false => Ok(()),
4646+ },
4747+ None => Ok(()),
4848+ }
4949+ }
5050+}
5151+5252+pub async fn run(args: Args) -> Result<()> {
5353+ args.validate()?;
5454+5555+ let dest_db = db::Instance::new(&args.destination, true).await?;
5656+5757+ if let Some(enabled) = args.enabled {
5858+ dest_db.set_transcoding(enabled).await?;
5959+6060+ if !enabled {
6161+ return Ok(()); // user is disabling transcoding, we can return early
6262+ }
6363+ }
6464+6565+ dest_db
6666+ .set_transcoding_settings(
6767+ args.codec.map(|e| e.to_string()),
6868+ args.bitrate.map(|e| e.to_string()),
6969+ args.sampling_rate.map(|e| e.to_string()),
7070+ )
7171+ .await?;
7272+7373+ println!("{:?}", dest_db.get_transcoding_settings().await?);
7474+7575+ Ok(())
7676+}
···51515252/// Returns true if name has one of the supported music file extension.
5353fn is_music(name: &String) -> bool {
5454+ // TODO: look at the mimetype instead of trusting the extension
5455 let formats = ["flac", "mp3", "ogg", "mp4", "m4a"];
55565657 formats