Refactor

- Removed `-z` zippyshare resolver
- Automatically detect zippyshare urls and resolve them

Cargo.lock

@@ -11,6 +11,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -170,6 +176,7 @@ dependencies = [
 name = "ffdl"
 version = "0.1.2"
 dependencies = [
+ "anyhow",
  "chrono",
  "clap",
  "crossterm",
@@ -177,6 +184,7 @@ dependencies = [
  "percent-encoding",
  "regex",
  "reqwest",
+ "thiserror",
  "tokio",
 ]
 
@@ -983,6 +991,26 @@ version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
 
+[[package]]
+name = "thiserror"
+version = "1.0.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "time"
 version = "0.1.44"

Cargo.toml

@@ -14,3 +14,5 @@ regex = "1.5.5"
 crossterm = "0.23.1"
 clap = { version = "3.1.6", features = [ "derive" ] }
 chrono = "0.4.19"
+thiserror = "1.0.30"
+anyhow = "1.0.56"

src/args.rs

@@ -18,14 +18,6 @@ pub struct CLIArgs {
     )]
     pub outdir: PathBuf,
 
-    #[clap(
-        short = 'i', 
-        long = "into-file", 
-        value_name = "FILENAME", 
-        help = "Force filename. This only works for single file downloads",
-    )]
-    pub into_file: Option<PathBuf>,
-
     #[clap(
         short = 'n',
         long = "num-files",
@@ -47,14 +39,6 @@ pub struct CLIArgs {
     )]
     pub conn_count: NonZeroU32,
 
-    #[clap(
-        short = 'z', 
-        long = "zippy", 
-        help = "The provided URLs are zippyshare URLs and need to be \
-            resolved to direct download urls",
-    )]
-    pub zippy: bool,
-
     #[clap( 
         short = 'l',
         long = "listfile",

src/dlreport.rs

@@ -9,7 +9,7 @@ use crossterm::execute;
 use crossterm::style::Print;
 use crossterm::terminal::{Clear, ClearType};
 
-use crate::errors::*;
+use anyhow::Result;
 
 #[derive(Clone, Debug)]
 pub enum DlStatus {
@@ -35,10 +35,7 @@ pub struct DlReporter {
 
 impl DlReporter {
     pub fn new(id: u32, transmitter: mpsc::UnboundedSender<DlReport>) -> DlReporter {
-        DlReporter {
-            id: id,
-            transmitter: transmitter,
-        }
+        DlReporter { id, transmitter }
     }
 
     pub fn send(&self, status: DlStatus) {
@@ -50,6 +47,46 @@ impl DlReporter {
             })
             .unwrap();
     }
+
+    pub fn init(&self, bytes_total: u64, filename: String) {
+        self.send(DlStatus::Init {
+            bytes_total,
+            filename,
+        })
+    }
+
+    pub fn update(&self, speed_mbps: f32, bytes_curr: u64) {
+        self.send(DlStatus::Update {
+            speed_mbps,
+            bytes_curr,
+        })
+    }
+
+    pub fn done(&self, duration_ms: u64) {
+        self.send(DlStatus::Done { duration_ms })
+    }
+
+    pub fn done_err(&self, filename: String) {
+        self.send(DlStatus::DoneErr { filename })
+    }
+
+    pub fn skipped(&self) {
+        self.send(DlStatus::Skipped);
+    }
+
+    pub fn msg(&self, msg: String) {
+        self.send(DlStatus::Message(msg));
+    }
+}
+
+#[macro_export]
+macro_rules! report_msg {
+    ($rep:ident, $fmt:expr) => {
+        DlReporter::msg(&$rep, $fmt.to_string());
+    };
+    ($rep:ident, $fmt:expr, $($fmt2:expr),+) => {
+        DlReporter::msg(&$rep, format!($fmt, $($fmt2,)+));
+    };
 }
 
 struct InfoHolder {
@@ -76,7 +113,7 @@ fn print_accumulated_report(
     moved_lines: u16,
     file_count_completed: i32,
     file_count_total: i32,
-) -> ResBE<u16> {
+) -> Result<u16> {
     let mut dl_speed_sum = 0.0;
 
     execute!(
@@ -98,12 +135,12 @@ fn print_accumulated_report(
 
     execute!(
         stdout(),
-        Print(format!("----------------------------------------")),
+        Print("----------------------------------------".to_string()),
         Clear(ClearType::UntilNewLine),
         Print("\n")
     )?;
 
-    for (_k, v) in statuses {
+    for v in statuses.values() {
         let percent_complete = v.progress as f64 / v.total_size as f64 * 100.0;
 
         execute!(
@@ -142,7 +179,7 @@ fn print_accumulated_report(
 pub async fn watch_and_print_reports(
     mut receiver: mpsc::UnboundedReceiver<DlReport>,
     file_count_total: i32,
-) -> ResBE<()> {
+) -> Result<()> {
     let mut statuses: HashMap<u32, InfoHolder> = HashMap::new();
     let mut moved_lines = 0;
     let mut msg_queue = VecDeque::new();

src/download.rs

@@ -1,3 +1,4 @@
+use anyhow::Result;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
 use percent_encoding::percent_decode_str;
@@ -24,7 +25,7 @@ impl RollingAverage {
     }
 
     fn value(&self) -> f64 {
-        if self.data.len() == 0 {
+        if self.data.is_empty() {
             0.0
         } else {
             let mut max = self.data[0];
@@ -63,7 +64,7 @@ impl RollingAverage {
 
 /// Get the filename at the end of the given URL. This will decode the URL Encoding.
 pub fn url_to_filename(url: &str) -> String {
-    let url_dec = percent_decode_str(&url)
+    let url_dec = percent_decode_str(url)
         .decode_utf8_lossy()
         .to_owned()
         .to_string();
@@ -73,7 +74,7 @@ pub fn url_to_filename(url: &str) -> String {
         .to_str()
         .unwrap();
     // Split at ? and return the first part. If no ? is present, this just returns the full string
-    file_name.split("?").next().unwrap().to_string()
+    file_name.split('?').next().unwrap().to_string()
 }
 
 pub async fn download_feedback(
@@ -81,7 +82,7 @@ pub async fn download_feedback(
     into_file: &Path,
     rep: DlReporter,
     content_length: Option<u64>,
-) -> ResBE<()> {
+) -> Result<()> {
     download_feedback_chunks(url, into_file, rep, None, content_length).await
 }
 
@@ -91,13 +92,10 @@ pub async fn download_feedback_chunks(
     rep: DlReporter,
     from_to: Option<(u64, u64)>,
     content_length: Option<u64>,
-) -> ResBE<()> {
+) -> Result<()> {
     let mut content_length = match content_length {
         Some(it) => it,
-        None => {
-            let (content_length, _) = http_get_filesize_and_range_support(url).await?;
-            content_length
-        }
+        None => http_get_filesize_and_range_support(url).await?.filesize,
     };
 
     // Send the HTTP request to download the given link
@@ -122,7 +120,7 @@ pub async fn download_feedback_chunks(
     let mut ofile = opts
         .create(true)
         .write(true)
-        .truncate(!from_to.is_some())
+        .truncate(from_to.is_none())
         .open(into_file)
         .await?;
 
@@ -133,10 +131,7 @@ pub async fn download_feedback_chunks(
     let filename = into_file.file_name().unwrap().to_str().unwrap();
 
     // Report the download start
-    rep.send(DlStatus::Init {
-        bytes_total: content_length,
-        filename: filename.to_string(),
-    });
+    rep.init(content_length, filename.to_string());
 
     let mut curr_progress = 0;
     let mut speed_mbps = 0.0;
@@ -189,13 +184,10 @@ pub async fn download_feedback_chunks(
         }
 
         // Send status update report
-        rep.send(DlStatus::Update {
-            speed_mbps,
-            bytes_curr: curr_progress,
-        });
+        rep.update(speed_mbps, curr_progress);
     }
 
-    if buff.len() > 0 {
+    if !buff.is_empty() {
         ofile.write_all(&buff).await?;
     }
 
@@ -209,7 +201,7 @@ pub async fn download_feedback_chunks(
     let duration_ms = t_start.elapsed()?.as_millis() as u64;
 
     // Send report that the download is finished
-    rep.send(DlStatus::Done { duration_ms });
+    rep.done(duration_ms);
 
     Ok(())
 }
@@ -222,10 +214,10 @@ pub async fn download_feedback_multi(
     rep: DlReporter,
     conn_count: u32,
     content_length: Option<u64>,
-) -> ResBE<()> {
+) -> Result<()> {
     let content_length = match content_length {
         Some(it) => it,
-        None => http_get_filesize_and_range_support(url).await?.0,
+        None => http_get_filesize_and_range_support(url).await?.filesize,
     };
 
     // Create zeroed file with 1 byte too much. This will be truncated on download
@@ -242,8 +234,8 @@ pub async fn download_feedback_multi(
     let t_start = SystemTime::now();
 
     for index in 0..conn_count {
-        let url = url.clone().to_owned();
-        let into_file = into_file.clone().to_owned();
+        let url = url.to_owned();
+        let into_file = into_file.to_owned();
 
         let tx = tx.clone();
 
@@ -269,7 +261,6 @@ pub async fn download_feedback_multi(
                 Some(specific_content_length),
             )
             .await
-            .map_err(|e| e.to_string())
         }))
     }
 
@@ -277,14 +268,11 @@ pub async fn download_feedback_multi(
 
     let filename = Path::new(into_file).file_name().unwrap().to_str().unwrap();
 
-    rep.send(DlStatus::Init {
-        bytes_total: content_length,
-        filename: filename.to_string(),
-    });
+    rep.init(content_length, filename.to_string());
 
     let rep_task = rep.clone();
 
-    let mut t_last = t_start.clone();
+    let mut t_last = t_start;
 
     let manager_handle = tokio::task::spawn(async move {
         let rep = rep_task;
@@ -322,10 +310,7 @@ pub async fn download_feedback_multi(
                         t_last = SystemTime::now();
                     }
 
-                    rep.send(DlStatus::Update {
-                        speed_mbps: speed_mbps,
-                        bytes_curr: progress_curr,
-                    });
+                    rep.update(speed_mbps, progress_curr);
                 }
                 DlStatus::Done { duration_ms: _ } => {
 
@@ -353,7 +338,7 @@ pub async fn download_feedback_multi(
 
             tokio::fs::remove_file(&into_file).await?;
 
-            return Err(e.into());
+            return Err(e);
         }
     }
 
@@ -369,50 +354,53 @@ pub async fn download_feedback_multi(
 
     ofile.set_len(content_length).await?;
 
-    rep.send(DlStatus::Done {
-        duration_ms: t_start.elapsed()?.as_millis() as u64,
-    });
+    rep.done(t_start.elapsed()?.as_millis() as u64);
 
     Ok(())
 }
 
-async fn create_zeroed_file(file: &Path, filesize: usize) -> ResBE<()> {
+async fn create_zeroed_file(file: &Path, filesize: usize) -> Result<()> {
     let ofile = tokio::fs::OpenOptions::new()
         .create(true)
-        // Open in write mode
         .write(true)
-        // Delete and overwrite the file
         .truncate(true)
         .open(file)
         .await?;
 
     ofile.set_len(filesize as u64).await?;
-
     Ok(())
 }
 
-pub async fn http_get_filesize_and_range_support(url: &str) -> ResBE<(u64, bool)> {
+pub struct HttpFileInfo {
+    pub filesize: u64,
+    pub range_support: bool,
+    pub filename: String,
+}
+
+pub async fn http_get_filesize_and_range_support(url: &str) -> Result<HttpFileInfo> {
     let resp = reqwest::Client::new().head(url).send().await?;
 
-    if let Some(filesize) = resp.headers().get(reqwest::header::CONTENT_LENGTH) {
-        if let Ok(val_str) = filesize.to_str() {
-            if let Ok(val) = val_str.parse::<u64>() {
-                let mut range_supported = false;
+    let filesize = resp
+        .headers()
+        .get(reqwest::header::CONTENT_LENGTH)
+        .and_then(|it| it.to_str().unwrap().parse::<u64>().ok())
+        .ok_or(DlError::ContentLengthUnknown)?;
 
-                if let Some(range) = resp.headers().get(reqwest::header::ACCEPT_RANGES) {
-                    if let Ok(range) = range.to_str() {
-                        if range == "bytes" {
-                            range_supported = true;
-                        }
-                    }
-                }
+    let range = resp
+        .headers()
+        .get(reqwest::header::ACCEPT_RANGES)
+        .and_then(|it| it.to_str().ok());
+    let range_support = matches!(range, Some("bytes"));
 
-                return Ok((val, range_supported));
-            }
-        }
-    }
+    let filename = url_to_filename(url);
 
-    Err(DlError::ContentLengthUnknown.into())
+    let info = HttpFileInfo {
+        filesize,
+        range_support,
+        filename,
+    };
+
+    Ok(info)
 }
 
 #[cfg(test)]

src/errors.rs

@@ -1,27 +1,14 @@
-use std::error::Error;
-use std::fmt::{self, Display, Formatter};
-
-/// Result Boxed Error
-pub type ResBE<T> = Result<T, Box<dyn Error>>;
+use thiserror::Error;
 
 #[allow(unused)]
-#[derive(Clone, Debug)]
+#[derive(Error, Clone, Debug)]
 pub enum DlError {
+    #[error("Bad http response status")]
     BadHttpStatus,
+    #[error("Content-Length is unknown")]
     ContentLengthUnknown,
+    #[error("Http server sent no more data")]
     HttpNoData,
+    #[error("Unknown download error: '{0}'")]
     Other(String),
 }
-
-impl Error for DlError {}
-
-impl Display for DlError {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        match self {
-            DlError::BadHttpStatus => write!(f, "Bad http response status"),
-            DlError::ContentLengthUnknown => write!(f, "Content-Length is unknown"),
-            DlError::HttpNoData => write!(f, "Http server sent no more data"),
-            DlError::Other(s) => write!(f, "Unknown download error: '{}'", s),
-        }
-    }
-}

src/main.rs

@@ -7,7 +7,6 @@ use std::{
 };
 
 use clap::Parser;
-use download::{download_feedback, download_feedback_multi, http_get_filesize_and_range_support};
 use futures::future::join_all;
 use tokio::{
     fs::create_dir_all,
@@ -19,10 +18,13 @@ use tokio::{
 
 use crate::{
     args::CLIArgs,
-    dlreport::{watch_and_print_reports, DlReport, DlReporter, DlStatus},
-    errors::ResBE,
+    dlreport::{watch_and_print_reports, DlReport, DlReporter},
+    download::{download_feedback, download_feedback_multi, http_get_filesize_and_range_support},
+    zippy::is_zippyshare_url,
 };
 
+use anyhow::Result;
+
 mod args;
 mod dlreport;
 mod download;
@@ -37,7 +39,7 @@ struct DlRequest {
 type SyncQueue = Arc<Mutex<VecDeque<DlRequest>>>;
 
 #[tokio::main]
-async fn main() -> ResBE<()> {
+async fn main() -> Result<()> {
     let args = CLIArgs::parse();
 
     // Combine all urls taken from files and the ones provided on the command line
@@ -61,7 +63,7 @@ async fn main() -> ResBE<()> {
 }
 
 /// Parse a listfile and return all urls found in it
-async fn urls_from_listfile(listfile: &Path) -> ResBE<Vec<String>> {
+async fn urls_from_listfile(listfile: &Path) -> Result<Vec<String>> {
     let text = tokio::fs::read_to_string(listfile).await?;
     let urls = text
         .lines()
@@ -73,7 +75,7 @@ async fn urls_from_listfile(listfile: &Path) -> ResBE<Vec<String>> {
 }
 
 // Download all files in parallel according to the provided CLI arguments
-async fn download_multiple(args: CLIArgs, raw_urls: Vec<String>) -> ResBE<()> {
+async fn download_multiple(args: CLIArgs, raw_urls: Vec<String>) -> Result<()> {
     let num_urls = raw_urls.len();
     let urls: SyncQueue = Default::default();
 
@@ -113,24 +115,19 @@ async fn download_multiple(args: CLIArgs, raw_urls: Vec<String>) -> ResBE<()> {
 }
 
 async fn download_job(urls: SyncQueue, reporter: UnboundedSender<DlReport>, cli_args: CLIArgs) {
-    loop {
-        // Get the next url to download or break if there are no more urls
-        let dlreq = match urls.lock().await.pop_front() {
-            Some(it) => it,
-            None => break,
-        };
-
+    while let Some(dlreq) = urls.lock().await.pop_front() {
         let reporter = DlReporter::new(dlreq.id as u32, reporter.clone());
 
         // Resolve the zippy url to the direct download url if necessary
-        let url = if cli_args.zippy {
+        let url = if is_zippyshare_url(&dlreq.url) {
             match zippy::resolve_link(&dlreq.url).await {
                 Ok(url) => url,
                 Err(_e) => {
-                    reporter.send(DlStatus::Message(format!(
-                        "Zippyshare link could not be resolved: {}",
+                    report_msg!(
+                        reporter,
+                        "Zippyshare link could not be resolved, skipping: {}",
                         dlreq.url
-                    )));
+                    );
                     continue;
                 }
             }
@@ -138,72 +135,64 @@ async fn download_job(urls: SyncQueue, reporter: UnboundedSender<DlReport>, cli_
             dlreq.url.to_string()
         };
 
-        let file_name = cli_args
-            .into_file
-            .clone()
-            .unwrap_or_else(|| download::url_to_filename(&url).into());
+        let info = match http_get_filesize_and_range_support(&url).await {
+            Ok(it) => it,
+            Err(_e) => {
+                report_msg!(reporter, "Error while querying metadata: {url}");
+                continue;
+            }
+        };
 
         let into_file: PathBuf = cli_args
             .outdir
-            .join(Path::new(&file_name))
+            .join(Path::new(&info.filename))
             .to_str()
             .unwrap()
             .to_string()
             .into();
 
-        let (filesize, range_supported) = match http_get_filesize_and_range_support(&url).await {
-            Ok((filesize, range_supported)) => (filesize, range_supported),
-            Err(_e) => {
-                reporter.send(DlStatus::Message(format!(
-                    "Error while querying metadata: {}",
-                    url
-                )));
-                continue;
-            }
-        };
-
         // If file with same name is present locally, check filesize
         if into_file.exists() {
             let local_filesize = std::fs::metadata(&into_file).unwrap().len();
 
-            if filesize == local_filesize {
-                reporter.send(DlStatus::Message(format!(
+            if info.filesize == local_filesize {
+                report_msg!(
+                    reporter,
                     "Skipping file '{}': already present",
-                    file_name.display()
-                )));
-                reporter.send(DlStatus::Skipped);
+                    info.filename
+                );
+                reporter.skipped();
                 continue;
             } else {
-                reporter.send(DlStatus::Message(format!(
+                report_msg!(
+                    reporter,
                     "Replacing file '{}': present but not completed",
-                    &file_name.display()
-                )));
+                    &info.filename
+                );
             }
         }
 
         let dl_status = if cli_args.conn_count.get() == 1 {
-            download_feedback(&url, &into_file, reporter.clone(), Some(filesize)).await
-        } else if !range_supported {
-            reporter.send(DlStatus::Message(format!(
-                "Server does not support range headers. Downloading with single connection: {}",
-                url
-            )));
-            download_feedback(&url, &into_file, reporter.clone(), Some(filesize)).await
+            download_feedback(&url, &into_file, reporter.clone(), Some(info.filesize)).await
+        } else if !info.range_support {
+            report_msg!(
+                reporter,
+                "Server does not support range headers. Downloading with single connection: {url}"
+            );
+            download_feedback(&url, &into_file, reporter.clone(), Some(info.filesize)).await
         } else {
             download_feedback_multi(
                 &url,
                 &into_file,
                 reporter.clone(),
                 cli_args.conn_count.get(),
-                Some(filesize),
+                Some(info.filesize),
             )
             .await
         };
 
         if dl_status.is_err() {
-            reporter.send(DlStatus::DoneErr {
-                filename: file_name.to_str().unwrap().to_string(),
-            });
+            reporter.done_err(info.filename);
         }
     }
 }

src/zippy.rs

@@ -1,7 +1,12 @@
+use anyhow::Result;
 use regex::Regex;
 use std::io::{Error, ErrorKind};
 
-use crate::errors::ResBE;
+pub fn is_zippyshare_url(url: &str) -> bool {
+    Regex::new(r"^https?://(?:www\d*\.)?zippyshare\.com/v/[0-9a-zA-Z]+/file\.html$")
+        .unwrap()
+        .is_match(url)
+}
 
 /*
 Updated: 07.03.2022
@@ -17,15 +22,15 @@ Link generation code:
 document.getElementById('dlbutton').href = "/d/0Ky7p1C6/" + (186549 % 51245 + 186549 % 913) + "/some-file-name.part1.rar";
 ```
 */
-pub async fn resolve_link(url: &str) -> ResBE<String> {
+pub async fn resolve_link(url: &str) -> Result<String> {
     // Regex to check if the provided url is a zippyshare download url
     let re = Regex::new(r"(https://www\d*\.zippyshare\.com)")?;
-    if !re.is_match(&url) {
+    if !re.is_match(url) {
         return Err(Error::new(ErrorKind::Other, "URL is not a zippyshare url").into());
     }
 
     // Extract the hostname (with https:// prefix) for later
-    let base_host = &re.captures(&url).unwrap()[0];
+    let base_host = &re.captures(url).unwrap()[0];
 
     // Download the html body for the download page
     let body = reqwest::get(url).await?.text().await?;
@@ -42,10 +47,10 @@ pub async fn resolve_link(url: &str) -> ResBE<String> {
 
     let url_start = &cap_link[1];
     let url_end = &cap_link[5];
-    let n2: i32 = i32::from_str_radix(&cap_link[2], 10)?;
-    let n3: i32 = i32::from_str_radix(&cap_link[3], 10)?;
+    let n2: i32 = cap_link[2].parse()?;
+    let n3: i32 = cap_link[3].parse()?;
     let n4 = n2;
-    let n5: i32 = i32::from_str_radix(&cap_link[4], 10)?;
+    let n5: i32 = cap_link[4].parse()?;
 
     let mixed = n2 % n3 + n4 % n5;