use std::path::Path; use std::process::exit; use clap::{ App, Arg, ArgGroup, crate_version }; use tokio::sync::mpsc; use futures::future::join_all; use std::io::BufRead; use std::time::SystemTime; use dlreport::{ DlReport, DlStatus, DlReporter }; use errors::ResBE; mod zippy; mod download; mod errors; mod dlreport; #[tokio::main] async fn main() -> ResBE<()> { let arguments = App::new("FDL - Fast/File Downloader") .version(crate_version!()) .about("Download files fast") .arg( Arg::with_name("outdir") .short("o") .long("outdir") .value_name("OUTPUT DIR") .takes_value(true) .help("Set the output directory") ) .arg( Arg::with_name("numdl") .short("n") .long("numdl") .value_name("NUMBER OF CONCURRENT DOWNLOADS") .takes_value(true) .help("Specify the number concurrent downloads") ) .arg( Arg::with_name("boost") .short("b") .long("boost") .value_name("CONNECTIONS PER FILE") .takes_value(true) .help("Specify the number connections per single downloads. \ Files started with boost can't be continued. \ NOTE: This will likely cause IO bottlenecks on HDDs") ) .arg( Arg::with_name("zippyshare") .short("z") .long("zippy") .takes_value(false) .help("The provided URLs are zippyshare URLs and need to be resolved") ) .group( ArgGroup::with_name("action") .required(true) ) .arg( Arg::with_name("listfile") .short("l") .long("listfile") .value_name("URL LIST") .takes_value(true) .group("action") .help("Download all files form the specified url list") ) .arg( Arg::with_name("download") .short("d") .long("download") .value_name("URL") .takes_value(true) .group("action") .help("Download only the specified URL") ) .arg( Arg::with_name("resolve") .short("r") .long("resolve") .value_name("URL") .takes_value(true) .group("action") .help("Resolve the zippyshare url to real download url") ) .get_matches(); let outdir = match arguments.value_of("outdir") { Some(it) => it, None => "./" }; let numparal = match arguments.value_of("numdl") { Some(it) => it, None => "1" }; let numparal: i32 = match numparal.parse() { Ok(it) => it, Err(_) => { eprintln!("Invalid value for numdl: {}", numparal); exit(1); } }; let boost = match arguments.value_of("boost") { Some(it) => it, None => "1" }; let boost: i32 = match boost.parse() { Ok(it) => it, Err(_) => { eprintln!("Invalid value for boost: {}", numparal); exit(1); } }; let is_zippy = arguments.is_present("zippyshare"); if arguments.is_present("listfile") { let listfile = arguments.value_of("listfile").unwrap(); let ifile = std::fs::File::open(listfile)?; let urls: Vec = std::io::BufReader::new(ifile) .lines() .map(|l| l.unwrap()) .filter(|url| url.len() > 0 && !url.starts_with("#")) .collect(); download_multiple(urls, outdir, numparal, boost, is_zippy).await?; } else if arguments.is_present("download") { let url = arguments.value_of("download").unwrap(); let numparal = if boost != 1 { boost } else { numparal }; download_multiple(vec![url.to_string()], outdir, 1, numparal, is_zippy).await?; } else if arguments.is_present("resolve") { let url = arguments.value_of("resolve").unwrap(); match zippy::resolve_link(&url).await { Ok(resolved_url) => { println!("{}", resolved_url); }, Err(e) => { println!("Zippyshare link could not be resolved"); eprintln!("{}", e); exit(1); } } } else { println!("Something went very wrong..."); } Ok(()) } async fn download_multiple(urls: Vec, outdir: &str, numparal: i32, boost: i32, is_zippy: bool) -> ResBE<()> { let outdir = Path::new(outdir); if !outdir.exists() { std::fs::create_dir_all(outdir)?; } let t_start = SystemTime::now(); let mut joiners = Vec::new(); let (tx, rx) = mpsc::unbounded_channel::(); for offset in 0..numparal { let urls: Vec = urls .iter() .enumerate() .filter(|(index, _)| (index) % numparal as usize == offset as usize) .map(|(_, v)| v.to_owned()) .collect(); let tx = tx.clone(); let outdir = outdir.to_owned(); let offset = offset; joiners.push(tokio::task::spawn(async move { for (i, url) in urls.iter().enumerate() { let tx = tx.clone(); // Recalculated index in the main url vector, used as id let global_url_index = i as i32 * numparal + offset; let rep = DlReporter::new(global_url_index, tx); let url = if is_zippy { match zippy::resolve_link(&url).await { Ok(url) => url, Err(_e) => { rep.send( DlStatus::MessageNow(format!("Zippyshare link could not be resolved: {}", url)) ).unwrap(); continue; } } } else { url.to_string() }; let file_name = download::url_to_filename(&url); let into_file = outdir.join(Path::new(&file_name)) .to_str().unwrap().to_string(); let path_into_file = Path::new(&into_file); // If file with same name is present locally, check filesize if path_into_file.exists() { let (filesize, _) = download::http_get_filesize_and_range_support(&url).await.unwrap(); let local_filesize = std::fs::metadata(path_into_file).unwrap().len(); if filesize == local_filesize { rep.send(DlStatus::MessageNow(format!("Skipping file '{}': already present", &file_name))).unwrap(); continue; } else { rep.send(DlStatus::MessageNow(format!("Replacing file '{}': present but not completed", &file_name))).unwrap(); } } if boost == 1 { if let Err(_e) = download::download_feedback(&url, &into_file, rep.clone()).await { rep.send(DlStatus::DoneErr { filename: into_file.to_string() }).unwrap(); } } else { if let Err(_e) = download::download_feedback_multi(&url, &into_file, rep.clone(), boost).await { rep.send(DlStatus::DoneErr { filename: into_file.to_string() }).unwrap(); } }; } })) } drop(tx); dlreport::watch_and_print_reports(rx).await?; join_all(joiners).await; println!("Total time: {}s", t_start.elapsed()?.as_secs()); Ok(()) }