Update crates + rewrite arg parsing + refactor

- Updated all crates to the latest versions
- Rewrote arg parsing using clap derive
- Started refactoring arg parsing & code in main

Cargo.toml

@@ -3,13 +3,14 @@ name = "ffdl"
 version = "0.1.2"
 authors = ["daniel m <danielm@dnml.de>"]
 edition = "2018"
+description = "Download files fast"
 
 [dependencies]
-tokio = { version = "1.2.0", features = [ "full" ] }
+tokio = { version = "1.17.0", features = [ "full" ] }
-reqwest = { version = "0.11.2", features = [ "stream" ] }
+reqwest = { version = "0.11.10", features = [ "stream" ] }
-futures = "0.3.12"
+futures = "0.3.21"
 percent-encoding = "2.1.0"
-regex = "1.4.3"
+regex = "1.5.5"
-crossterm = "0.19.0"
+crossterm = "0.23.1"
-clap = "2.33.3"
+clap = { version = "3.1.6", features = [ "derive" ] }
-chrono = "0.4"
+chrono = "0.4.19"

src/args.rs

@@ -0,0 +1,94 @@
+use std::num::NonZeroU32;
+use clap::{Parser, ArgGroup};
+
+#[derive(Clone, Debug)]
+pub enum CLIAction {
+    DownloadUrl(String),
+    ResolveZippyUrl(String),
+    UrlList(String),
+}
+
+#[derive(Parser, Clone, Debug)]
+#[clap(
+    version, 
+    about, 
+    long_about = None, 
+    name = "FFDL - Fast File Downloader",
+    group(
+        ArgGroup::new("action")
+            .required(true)
+            .args(&["listfile", "download", "zippy-resolve"])
+    )
+)]
+pub struct CLIArgs {
+    #[clap(
+        short, 
+        long, 
+        value_name = "OUTPUT DIR", 
+        default_value = "./", 
+        help = "Set the output directory. The directory will be created if it doesn't exit yet"
+    )]
+    pub outdir: String,
+
+    #[clap(
+        short, 
+        long = "into-file", 
+        value_name = "FILENAME", 
+        help = "Force filename. This only works for single file downloads",
+        requires = "download"
+    )]
+    pub into_file: Option<String>,
+
+    #[clap(
+        short = 'n',
+        long = "num-files",
+        value_name = "NUMBER OF CONCURRENT FILE DOWNLOADS",
+        default_value = "1",
+        help = "Specify the number of concurrent downloads",
+        requires = "listfile"
+    )]
+    pub file_count: NonZeroU32,
+
+    #[clap(
+        short, 
+        long = "connections",
+        value_name = "NUMBER OF CONCURRENT CONNECTIONS", 
+        default_value = "1",
+        help = "The number concurrent connections per file download. \
+            Downloads might fail when the number of connections is too high. \
+            Files started with multiple connections currently can't be continued. \
+            NOTE: This will likely cause IO bottlenecks on HDDs"
+        )]
+        pub conn_count: NonZeroU32,
+
+    #[clap(
+        short, 
+        long, 
+        help = "The provided URLs are zippyshare URLs and need to be \
+            resolved to direct download urls"
+    )]
+    pub zippy: bool,
+
+    #[clap(	
+        short = 'l',
+        long = "listfile",
+        value_name = "URL LISTFILE",
+        help = "Download all files from the specified url list file",
+    )]
+    pub listfile: Option<String>,
+
+    #[clap(
+        short = 'd',
+        long = "download",
+        value_name = "URL",
+        help = "Download only the one file fromn the specified url",
+    )]
+    pub download: Option<String>,
+
+    #[clap(
+        long = "zippy-resolve",
+        value_name = "ZIPPYSHARE URL",
+        help = "Resolve the zippy share url to the direct download url",
+    )]
+    pub zippy_resolve: Option<String>,
+}

src/main.rs

@@ -1,255 +1,91 @@
-use std::collections::VecDeque;
+use std::{
-use std::path::Path;
+    collections::VecDeque, io::BufRead, path::Path, process::exit, sync::Arc, sync::Mutex,
-use std::process::exit;
+    time::SystemTime,
-use std::sync::Arc;
+};
-use clap::{ App, Arg, ArgGroup, crate_version };
+
-use std::sync::Mutex;
+use clap::Parser;
-use tokio::sync::mpsc;
 use futures::future::join_all;
-use std::io::BufRead;
+use tokio::sync::mpsc;
-use std::time::SystemTime;
 
-use dlreport::{ DlReport, DlStatus, DlReporter };
+use crate::{
-use errors::ResBE;
+    args::{CLIAction, CLIArgs},
+    dlreport::{DlReport, DlReporter, DlStatus},
+    errors::ResBE,
+};
 
-mod zippy;
+mod args;
+mod dlreport;
 mod download;
 mod errors;
-mod dlreport;
+mod zippy;
-
-
-#[derive(Clone, Debug)]
-enum CLIAction {
-    DownloadUrl(String),
-    ResolveZippyUrl(String),
-    UrlList(String),
-    None
-}
-
-#[derive(Clone, Debug)]
-struct CLIArguments {
-    outdir: String,
-    into_file: Option<String>,
-    parallel_file_count: u32,
-    conn_count: u32,
-    zippy: bool,
-    action: CLIAction,
-    urls: Vec<String>
-}
-
 
 #[tokio::main]
 async fn main() -> ResBE<()> {
+    let args = CLIArgs::parse();
 
-    let arguments = App::new("FFDL - Fast File Downloader")
+    let action = match (&args.listfile, &args.download, &args.zippy_resolve) {
-        .version(crate_version!())
+        (Some(listfile), None, None) => CLIAction::UrlList(listfile.to_string()),
-        .about("Download files fast")
+        (None, Some(url), None) => CLIAction::DownloadUrl(url.to_string()),
-        .arg(
+        (None, None, Some(zippy_url)) => CLIAction::ResolveZippyUrl(zippy_url.to_string()),
-            Arg::with_name("outdir")
+        _ => unreachable!(),
-                .short("o")
-                .long("outdir")
-                .value_name("OUTPUT DIR")
-                .takes_value(true)
-                .help("Set the output directory. The directory will be created \
-                        if it doesn't exit yet")
-        )
-        .arg(
-            Arg::with_name("into_file")
-                .short("i")
-                .long("into-file")
-                .value_name("FILENAME")
-                .takes_value(true)
-                .requires("download")
-                .help("Force filename. This only works for single file downloads")
-        )
-        .arg(
-            Arg::with_name("file_count")
-                .short("n")
-                .long("num-files")
-                .value_name("NUMBER OF CONCURRENT FILE DOWNLOADS")
-                .takes_value(true)
-                .help("Specify the number concurrent file downloads")
-        )
-        .arg(
-            Arg::with_name("conn_count")
-                .short("c")
-                .long("connections")
-                .value_name("NUMBER OF CONNECTIONS")
-                .takes_value(true)
-                .help("The number concurrent connections per file download. \
-                        Downloads might fail when the number of connections is \
-                        too high. Files started with multiple connections can't \
-                        be continued. NOTE: This will likely cause IO \
-                        bottlenecks on HDDs")
-        )
-        .arg(
-            Arg::with_name("zippyshare")
-                .short("z")
-                .long("zippy")
-                .takes_value(false)
-                .help("The provided URLs are zippyshare URLs and need to be resolved")
-        )
-        .group(
-            ArgGroup::with_name("action")
-                .required(true)
-        )
-        .arg(
-            Arg::with_name("listfile")
-                .short("l")
-                .long("listfile")
-                .value_name("URL LIST")
-                .takes_value(true)
-                .group("action")
-                .help("Download all files form the specified url list")
-        )
-        .arg(
-            Arg::with_name("download")
-                .short("d")
-                .long("download")
-                .value_name("URL")
-                .takes_value(true)
-                .group("action")
-                .help("Download only the specified URL")
-        )
-        .arg(
-            Arg::with_name("zippy-resolve")
-                .long("zippy-resolve")
-                .value_name("ZIPPYSHARE URL")
-                .takes_value(true)
-                .group("action")
-                .help("Resolve the zippyshare url to real download url")
-        )
-        .get_matches();
-    
-
-    let outdir = arguments.value_of("outdir").unwrap_or("./").to_string();
-
-    let into_file = arguments.value_of("into_file").map(String::from);
-
-    let file_count = arguments.value_of("file_count").unwrap_or("1");
-
-    let file_count: u32 = file_count.parse().unwrap_or_else(|_| {
-        eprintln!("Invalid value for num-files: {}", file_count);
-        exit(1);
-    });
-    
-    if file_count <= 0 {
-        eprintln!("Invalid value for num-files: {}", file_count);
-        exit(1);
-    }
-
-    let conn_count = arguments.value_of("conn_count").unwrap_or("1");
-    
-    let conn_count: u32 = conn_count.parse().unwrap_or_else(|_| {
-        eprintln!("Invalid value for connections: {}", conn_count);
-        exit(1);
-    });
-    
-    if conn_count <= 0 {
-        eprintln!("Invalid value for connections: {}", conn_count);
-        exit(1);
-    }
-
-    let is_zippy = arguments.is_present("zippyshare");
-
-
-    let action = 
-        if let Some(listfile) = arguments.value_of("listfile") { 
-            CLIAction::UrlList (
-                listfile.to_string()
-            )
-        } else if let Some(download_url) = arguments.value_of("download") { 
-            CLIAction::DownloadUrl(
-                download_url.to_string()
-            )
-        } else if let Some(resolve_url) = arguments.value_of("zippy-resolve") { 
-            CLIAction::ResolveZippyUrl(
-                resolve_url.to_string()
-            )
-        } 
-        else {
-            CLIAction::None
     };
 
-
+    let urls = match action {
-    let mut cli_args = CLIArguments {
+        CLIAction::DownloadUrl(url) => vec![url.clone()],
-        outdir: outdir,
+        CLIAction::UrlList(listfile) => read_urls_from_listfile(&listfile).await,
-        into_file: into_file,
+        CLIAction::ResolveZippyUrl(url) => resolve_zippy_url(&url).await,
-        parallel_file_count: file_count,
-        conn_count: conn_count,
-        zippy: is_zippy,
-        action: action,
-        urls: Vec::new()
     };
 
-    // Evaluate and execute the requested action. The 3 different actions are 
+    download_multiple(args, urls).await
-    // mutally exclusive, so only one of them will be executed
-
-    match &cli_args.action {
-
-        CLIAction::UrlList(listfile) => {
-
-            let p_listfile = Path::new(listfile);
-
-            if !p_listfile.is_file() {
-                eprintln!("Listfile '{}' does not exist!", &listfile);
-                exit(1);
 }
 
-            let ifile = std::fs::File::open(p_listfile)?;
+async fn resolve_zippy_url(url: &str) -> ! {
-    
+    let resolved_url = zippy::resolve_link(&url).await.unwrap_or_else(|_| {
-            cli_args.urls = std::io::BufReader::new(ifile)
-                .lines()
-                .map(|l| l.unwrap())
-                .filter(|url| url.len() > 0 && !url.starts_with("#"))
-                .collect();
-        },
-
-        CLIAction::DownloadUrl(url) => {
-            cli_args.urls = vec![url.clone()];
-        }
-
-        CLIAction::ResolveZippyUrl(url) => {
-            let resolved_url = zippy::resolve_link(url).await.unwrap_or_else(|_| {
         println!("Zippyshare link could not be resolved");
         exit(1);
     });
 
     println!("{}", resolved_url);
     exit(0);
-        },
+}
 
-        CLIAction::None => {
+async fn read_urls_from_listfile(listfile: &str) -> Vec<String> {
-            eprintln!("No action selected. This should not happen");
+    let p_listfile = Path::new(&listfile);
+
+    if !p_listfile.is_file() {
+        eprintln!("Listfile '{}' does not exist!", &listfile);
         exit(1);
     }
 
+    let ifile = std::fs::File::open(p_listfile).unwrap();
+
+    std::io::BufReader::new(ifile)
+        .lines()
+        .map(|l| l.unwrap())
+        .filter(|url| url.len() > 0 && !url.starts_with("#"))
+        .collect()
 }
 
-    download_multiple(cli_args).await
+async fn download_multiple(cli_args: CLIArgs, raw_urls: Vec<String>) -> ResBE<()> {
-
-}
-
-async fn download_multiple(cli_args: CLIArguments) -> ResBE<()> {
     let outdir = cli_args.outdir;
     let outdir = Path::new(&outdir);
 
-    let parallel_file_count = cli_args.parallel_file_count;
+    let parallel_file_count = cli_args.file_count.get();
-
+    let conn_count = cli_args.conn_count.get();
     let zippy = cli_args.zippy;
 
-    let conn_count = cli_args.conn_count;
-
     let urls = Arc::new(Mutex::new(VecDeque::<(u32, String)>::new()));
 
-    cli_args.urls.iter().enumerate().for_each(|(i, url)| {
+    raw_urls.iter().enumerate().for_each(|(i, url)| {
         urls.lock().unwrap().push_back((i as u32, url.clone()));
     });
 
-
     if !outdir.exists() {
         if let Err(_e) = std::fs::create_dir_all(outdir) {
-            eprintln!("Error creating output directory '{}'", outdir.to_str().unwrap());
+            eprintln!(
+                "Error creating output directory '{}'",
+                outdir.to_str().unwrap()
+            );
             exit(1);
         }
     }
@@ -261,7 +97,6 @@ async fn download_multiple(cli_args: CLIArguments) -> ResBE<()> {
     let (tx, rx) = mpsc::unbounded_channel::<DlReport>();
 
     for _offset in 0..parallel_file_count {
-        
         let tx = tx.clone();
         let outdir = outdir.to_owned();
         let arg_filename = cli_args.into_file.clone();
@@ -269,11 +104,10 @@ async fn download_multiple(cli_args: CLIArguments) -> ResBE<()> {
         let urls = urls.clone();
 
         joiners.push(tokio::task::spawn(async move {
-
             loop {
                 let (global_url_index, url) = match urls.lock().unwrap().pop_front() {
                     Some(it) => it,
-                    None => break
+                    None => break,
                 };
 
                 let tx = tx.clone();
@@ -284,9 +118,10 @@ async fn download_multiple(cli_args: CLIArguments) -> ResBE<()> {
                     match zippy::resolve_link(&url).await {
                         Ok(url) => url,
                         Err(_e) => {
-                            rep.send(
+                            rep.send(DlStatus::Message(format!(
-                                DlStatus::Message(format!("Zippyshare link could not be resolved: {}", url))
+                                "Zippyshare link could not be resolved: {}",
-                            );
+                                url
+                            )));
                             continue;
                         }
                     }
@@ -294,67 +129,87 @@ async fn download_multiple(cli_args: CLIArguments) -> ResBE<()> {
                     url.to_string()
                 };
 
-                let file_name = arg_filename.clone().unwrap_or_else(|| download::url_to_filename(&url));
+                let file_name = arg_filename
+                    .clone()
+                    .unwrap_or_else(|| download::url_to_filename(&url));
 
-
+                let into_file = outdir
-                let into_file = outdir.join(Path::new(&file_name))
+                    .join(Path::new(&file_name))
-                    .to_str().unwrap().to_string();
+                    .to_str()
+                    .unwrap()
+                    .to_string();
                 let path_into_file = Path::new(&into_file);
 
-                let (filesize, range_supported) = match download::http_get_filesize_and_range_support(&url).await {
+                let (filesize, range_supported) =
+                    match download::http_get_filesize_and_range_support(&url).await {
                         Ok((filesize, range_supported)) => (filesize, range_supported),
                         Err(_e) => {
-                        rep.send(
+                            rep.send(DlStatus::Message(format!(
-                            DlStatus::Message(format!("Error while querying metadata: {}", url))
+                                "Error while querying metadata: {}",
-                        );
+                                url
+                            )));
                             continue;
                         }
                     };
 
                 // If file with same name is present locally, check filesize
                 if path_into_file.exists() {
-
                     let local_filesize = std::fs::metadata(path_into_file).unwrap().len();
 
                     if filesize == local_filesize {
-                        rep.send(DlStatus::Message(format!("Skipping file '{}': already present", &file_name)));
+                        rep.send(DlStatus::Message(format!(
+                            "Skipping file '{}': already present",
+                            &file_name
+                        )));
                         rep.send(DlStatus::Skipped);
                         continue;
                     } else {
-                        rep.send(DlStatus::Message(format!("Replacing file '{}': present but not completed", &file_name)));
+                        rep.send(DlStatus::Message(format!(
+                            "Replacing file '{}': present but not completed",
+                            &file_name
+                        )));
                     }
                 }
 
                 if conn_count == 1 {
-                    if let Err(_e) = download::download_feedback(&url, &into_file, rep.clone(), Some(filesize)).await {
+                    if let Err(_e) =
+                        download::download_feedback(&url, &into_file, rep.clone(), Some(filesize))
+                            .await
+                    {
                         rep.send(DlStatus::DoneErr {
-                            filename: file_name.to_string()
+                            filename: file_name.to_string(),
                         });
                     }
                 } else {
-                    
                     if !range_supported {
-                        rep.send(
+                        rep.send(DlStatus::Message(format!(
-                            DlStatus::Message(format!("Error Server does not support range header: {}", url))
+                            "Error Server does not support range header: {}",
-                        );
+                            url
+                        )));
                         continue;
                     }
 
-                    if let Err(_e) = download::download_feedback_multi(&url, &into_file, rep.clone(), conn_count, Some(filesize)).await {
+                    if let Err(_e) = download::download_feedback_multi(
+                        &url,
+                        &into_file,
+                        rep.clone(),
+                        conn_count,
+                        Some(filesize),
+                    )
+                    .await
+                    {
                         rep.send(DlStatus::DoneErr {
-                            filename: file_name.to_string()
+                            filename: file_name.to_string(),
                         });
                     }
                 };
-
             }
         }))
-
     }
 
     drop(tx);
 
-    dlreport::watch_and_print_reports(rx, cli_args.urls.len() as i32).await?;
+    dlreport::watch_and_print_reports(rx, raw_urls.len() as i32).await?;
 
     join_all(joiners).await;