dnlmlr/ffdl
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Update crates + rewrite arg parsing + refactor

Browse Source 
- Updated all crates to the latest versions
- Rewrote arg parsing using clap derive
- Started refactoring arg parsing & code in main
This commit is contained in:
Daniel M 2022-03-28 01:16:58 +02:00
parent 29475dd3bd
commit 240a3ace42
4 changed files with 582 additions and 600 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

690
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

15
Cargo.toml
View File

@ -3,13 +3,14 @@ name = "ffdl"
version = "0.1.2"
authors = ["daniel m <danielm@dnml.de>"]
edition = "2018"
description = "Download files fast"
[dependencies]
tokio = { version = "1.2.0", features = [ "full" ] }
reqwest = { version = "0.11.2", features = [ "stream" ] }
futures = "0.3.12"
tokio = { version = "1.17.0", features = [ "full" ] }
reqwest = { version = "0.11.10", features = [ "stream" ] }
futures = "0.3.21"
percent-encoding = "2.1.0"
regex = "1.4.3"
crossterm = "0.19.0"
clap = "2.33.3"
chrono = "0.4"
regex = "1.5.5"
crossterm = "0.23.1"
clap = { version = "3.1.6", features = [ "derive" ] }
chrono = "0.4.19"

94
src/args.rs Normal file
View File

@ -0,0 +1,94 @@
use std::num::NonZeroU32;
use clap::{Parser, ArgGroup};
#[derive(Clone, Debug)]
pub enum CLIAction {
DownloadUrl(String),
ResolveZippyUrl(String),
UrlList(String),
}
#[derive(Parser, Clone, Debug)]
#[clap(
version,
about,
long_about = None,
name = "FFDL - Fast File Downloader",
group(
ArgGroup::new("action")
.required(true)
.args(&["listfile", "download", "zippy-resolve"])
)
)]
pub struct CLIArgs {
#[clap(
short,
long,
value_name = "OUTPUT DIR",
default_value = "./",
help = "Set the output directory. The directory will be created if it doesn't exit yet"
)]
pub outdir: String,
#[clap(
short,
long = "into-file",
value_name = "FILENAME",
help = "Force filename. This only works for single file downloads",
requires = "download"
)]
pub into_file: Option<String>,
#[clap(
short = 'n',
long = "num-files",
value_name = "NUMBER OF CONCURRENT FILE DOWNLOADS",
default_value = "1",
help = "Specify the number of concurrent downloads",
requires = "listfile"
)]
pub file_count: NonZeroU32,
#[clap(
short,
long = "connections",
value_name = "NUMBER OF CONCURRENT CONNECTIONS",
default_value = "1",
help = "The number concurrent connections per file download. \
Downloads might fail when the number of connections is too high. \
Files started with multiple connections currently can't be continued. \
NOTE: This will likely cause IO bottlenecks on HDDs"
)]
pub conn_count: NonZeroU32,
#[clap(
short,
long,
help = "The provided URLs are zippyshare URLs and need to be \
resolved to direct download urls"
)]
pub zippy: bool,
#[clap(
short = 'l',
long = "listfile",
value_name = "URL LISTFILE",
help = "Download all files from the specified url list file",
)]
pub listfile: Option<String>,
#[clap(
short = 'd',
long = "download",
value_name = "URL",
help = "Download only the one file fromn the specified url",
)]
pub download: Option<String>,
#[clap(
long = "zippy-resolve",
value_name = "ZIPPYSHARE URL",
help = "Resolve the zippy share url to the direct download url",
)]
pub zippy_resolve: Option<String>,
}

375
src/main.rs
View File

@ -1,255 +1,91 @@
use std::collections::VecDeque;
use std::path::Path;
use std::process::exit;
use std::sync::Arc;
use clap::{ App, Arg, ArgGroup, crate_version };
use std::sync::Mutex;
use tokio::sync::mpsc;
use std::{
collections::VecDeque, io::BufRead, path::Path, process::exit, sync::Arc, sync::Mutex,
time::SystemTime,
};
use clap::Parser;
use futures::future::join_all;
use std::io::BufRead;
use std::time::SystemTime;
use tokio::sync::mpsc;
use dlreport::{ DlReport, DlStatus, DlReporter };
use errors::ResBE;
use crate::{
args::{CLIAction, CLIArgs},
dlreport::{DlReport, DlReporter, DlStatus},
errors::ResBE,
};
mod zippy;
mod args;
mod dlreport;
mod download;
mod errors;
mod dlreport;
#[derive(Clone, Debug)]
enum CLIAction {
DownloadUrl(String),
ResolveZippyUrl(String),
UrlList(String),
None
}
#[derive(Clone, Debug)]
struct CLIArguments {
outdir: String,
into_file: Option<String>,
parallel_file_count: u32,
conn_count: u32,
zippy: bool,
action: CLIAction,
urls: Vec<String>
}
mod zippy;
#[tokio::main]
async fn main() -> ResBE<()> {
let args = CLIArgs::parse();
let arguments = App::new("FFDL - Fast File Downloader")
.version(crate_version!())
.about("Download files fast")
.arg(
Arg::with_name("outdir")
.short("o")
.long("outdir")
.value_name("OUTPUT DIR")
.takes_value(true)
.help("Set the output directory. The directory will be created \
if it doesn't exit yet")
)
.arg(
Arg::with_name("into_file")
.short("i")
.long("into-file")
.value_name("FILENAME")
.takes_value(true)
.requires("download")
.help("Force filename. This only works for single file downloads")
)
.arg(
Arg::with_name("file_count")
.short("n")
.long("num-files")
.value_name("NUMBER OF CONCURRENT FILE DOWNLOADS")
.takes_value(true)
.help("Specify the number concurrent file downloads")
)
.arg(
Arg::with_name("conn_count")
.short("c")
.long("connections")
.value_name("NUMBER OF CONNECTIONS")
.takes_value(true)
.help("The number concurrent connections per file download. \
Downloads might fail when the number of connections is \
too high. Files started with multiple connections can't \
be continued. NOTE: This will likely cause IO \
bottlenecks on HDDs")
)
.arg(
Arg::with_name("zippyshare")
.short("z")
.long("zippy")
.takes_value(false)
.help("The provided URLs are zippyshare URLs and need to be resolved")
)
.group(
ArgGroup::with_name("action")
.required(true)
)
.arg(
Arg::with_name("listfile")
.short("l")
.long("listfile")
.value_name("URL LIST")
.takes_value(true)
.group("action")
.help("Download all files form the specified url list")
)
.arg(
Arg::with_name("download")
.short("d")
.long("download")
.value_name("URL")
.takes_value(true)
.group("action")
.help("Download only the specified URL")
)
.arg(
Arg::with_name("zippy-resolve")
.long("zippy-resolve")
.value_name("ZIPPYSHARE URL")
.takes_value(true)
.group("action")
.help("Resolve the zippyshare url to real download url")
)
.get_matches();
let outdir = arguments.value_of("outdir").unwrap_or("./").to_string();
let into_file = arguments.value_of("into_file").map(String::from);
let file_count = arguments.value_of("file_count").unwrap_or("1");
let file_count: u32 = file_count.parse().unwrap_or_else(|_| {
eprintln!("Invalid value for num-files: {}", file_count);
exit(1);
});
if file_count <= 0 {
eprintln!("Invalid value for num-files: {}", file_count);
exit(1);
}
let conn_count = arguments.value_of("conn_count").unwrap_or("1");
let conn_count: u32 = conn_count.parse().unwrap_or_else(|_| {
eprintln!("Invalid value for connections: {}", conn_count);
exit(1);
});
if conn_count <= 0 {
eprintln!("Invalid value for connections: {}", conn_count);
exit(1);
}
let is_zippy = arguments.is_present("zippyshare");
let action =
if let Some(listfile) = arguments.value_of("listfile") {
CLIAction::UrlList (
listfile.to_string()
)
} else if let Some(download_url) = arguments.value_of("download") {
CLIAction::DownloadUrl(
download_url.to_string()
)
} else if let Some(resolve_url) = arguments.value_of("zippy-resolve") {
CLIAction::ResolveZippyUrl(
resolve_url.to_string()
)
}
else {
CLIAction::None
};
let mut cli_args = CLIArguments {
outdir: outdir,
into_file: into_file,
parallel_file_count: file_count,
conn_count: conn_count,
zippy: is_zippy,
action: action,
urls: Vec::new()
let action = match (&args.listfile, &args.download, &args.zippy_resolve) {
(Some(listfile), None, None) => CLIAction::UrlList(listfile.to_string()),
(None, Some(url), None) => CLIAction::DownloadUrl(url.to_string()),
(None, None, Some(zippy_url)) => CLIAction::ResolveZippyUrl(zippy_url.to_string()),
_ => unreachable!(),
};
// Evaluate and execute the requested action. The 3 different actions are
// mutally exclusive, so only one of them will be executed
match &cli_args.action {
CLIAction::UrlList(listfile) => {
let p_listfile = Path::new(listfile);
if !p_listfile.is_file() {
eprintln!("Listfile '{}' does not exist!", &listfile);
exit(1);
}
let ifile = std::fs::File::open(p_listfile)?;
cli_args.urls = std::io::BufReader::new(ifile)
.lines()
.map(|l| l.unwrap())
.filter(|url| url.len() > 0 && !url.starts_with("#"))
.collect();
},
CLIAction::DownloadUrl(url) => {
cli_args.urls = vec![url.clone()];
}
CLIAction::ResolveZippyUrl(url) => {
let resolved_url = zippy::resolve_link(url).await.unwrap_or_else(|_| {
println!("Zippyshare link could not be resolved");
exit(1);
});
println!("{}", resolved_url);
exit(0);
},
CLIAction::None => {
eprintln!("No action selected. This should not happen");
exit(1);
}
}
download_multiple(cli_args).await
let urls = match action {
CLIAction::DownloadUrl(url) => vec![url.clone()],
CLIAction::UrlList(listfile) => read_urls_from_listfile(&listfile).await,
CLIAction::ResolveZippyUrl(url) => resolve_zippy_url(&url).await,
};
download_multiple(args, urls).await
}
async fn download_multiple(cli_args: CLIArguments) -> ResBE<()> {
async fn resolve_zippy_url(url: &str) -> ! {
let resolved_url = zippy::resolve_link(&url).await.unwrap_or_else(|_| {
println!("Zippyshare link could not be resolved");
exit(1);
});
println!("{}", resolved_url);
exit(0);
}
async fn read_urls_from_listfile(listfile: &str) -> Vec<String> {
let p_listfile = Path::new(&listfile);
if !p_listfile.is_file() {
eprintln!("Listfile '{}' does not exist!", &listfile);
exit(1);
}
let ifile = std::fs::File::open(p_listfile).unwrap();
std::io::BufReader::new(ifile)
.lines()
.map(|l| l.unwrap())
.filter(|url| url.len() > 0 && !url.starts_with("#"))
.collect()
}
async fn download_multiple(cli_args: CLIArgs, raw_urls: Vec<String>) -> ResBE<()> {
let outdir = cli_args.outdir;
let outdir = Path::new(&outdir);
let parallel_file_count = cli_args.parallel_file_count;
let parallel_file_count = cli_args.file_count.get();
let conn_count = cli_args.conn_count.get();
let zippy = cli_args.zippy;
let conn_count = cli_args.conn_count;
let urls = Arc::new(Mutex::new(VecDeque::<(u32, String)>::new()));
cli_args.urls.iter().enumerate().for_each(|(i, url)| {
raw_urls.iter().enumerate().for_each(|(i, url)| {
urls.lock().unwrap().push_back((i as u32, url.clone()));
});
if !outdir.exists() {
if let Err(_e) = std::fs::create_dir_all(outdir) {
eprintln!("Error creating output directory '{}'", outdir.to_str().unwrap());
eprintln!(
"Error creating output directory '{}'",
outdir.to_str().unwrap()
);
exit(1);
}
}
@ -260,8 +96,7 @@ async fn download_multiple(cli_args: CLIArguments) -> ResBE<()> {
let (tx, rx) = mpsc::unbounded_channel::<DlReport>();
for _offset in 0 .. parallel_file_count {
for _offset in 0..parallel_file_count {
let tx = tx.clone();
let outdir = outdir.to_owned();
let arg_filename = cli_args.into_file.clone();
@ -269,11 +104,10 @@ async fn download_multiple(cli_args: CLIArguments) -> ResBE<()> {
let urls = urls.clone();
joiners.push(tokio::task::spawn(async move {
loop {
let (global_url_index, url) = match urls.lock().unwrap().pop_front() {
Some(it) => it,
None => break
None => break,
};
let tx = tx.clone();
@ -284,9 +118,10 @@ async fn download_multiple(cli_args: CLIArguments) -> ResBE<()> {
match zippy::resolve_link(&url).await {
Ok(url) => url,
Err(_e) => {
rep.send(
DlStatus::Message(format!("Zippyshare link could not be resolved: {}", url))
);
rep.send(DlStatus::Message(format!(
"Zippyshare link could not be resolved: {}",
url
)));
continue;
}
}
@ -294,67 +129,87 @@ async fn download_multiple(cli_args: CLIArguments) -> ResBE<()> {
url.to_string()
};
let file_name = arg_filename.clone().unwrap_or_else(|| download::url_to_filename(&url));
let file_name = arg_filename
.clone()
.unwrap_or_else(|| download::url_to_filename(&url));
let into_file = outdir.join(Path::new(&file_name))
.to_str().unwrap().to_string();
let into_file = outdir
.join(Path::new(&file_name))
.to_str()
.unwrap()
.to_string();
let path_into_file = Path::new(&into_file);
let (filesize, range_supported) = match download::http_get_filesize_and_range_support(&url).await {
Ok((filesize, range_supported)) => (filesize, range_supported),
Err(_e) => {
rep.send(
DlStatus::Message(format!("Error while querying metadata: {}", url))
);
continue;
}
};
let (filesize, range_supported) =
match download::http_get_filesize_and_range_support(&url).await {
Ok((filesize, range_supported)) => (filesize, range_supported),
Err(_e) => {
rep.send(DlStatus::Message(format!(
"Error while querying metadata: {}",
url
)));
continue;
}
};
// If file with same name is present locally, check filesize
if path_into_file.exists() {
let local_filesize = std::fs::metadata(path_into_file).unwrap().len();
if filesize == local_filesize {
rep.send(DlStatus::Message(format!("Skipping file '{}': already present", &file_name)));
rep.send(DlStatus::Message(format!(
"Skipping file '{}': already present",
&file_name
)));
rep.send(DlStatus::Skipped);
continue;
} else {
rep.send(DlStatus::Message(format!("Replacing file '{}': present but not completed", &file_name)));
rep.send(DlStatus::Message(format!(
"Replacing file '{}': present but not completed",
&file_name
)));
}
}
if conn_count == 1 {
if let Err(_e) = download::download_feedback(&url, &into_file, rep.clone(), Some(filesize)).await {
if let Err(_e) =
download::download_feedback(&url, &into_file, rep.clone(), Some(filesize))
.await
{
rep.send(DlStatus::DoneErr {
filename: file_name.to_string()
filename: file_name.to_string(),
});
}
} else {
if !range_supported {
rep.send(
DlStatus::Message(format!("Error Server does not support range header: {}", url))
);
rep.send(DlStatus::Message(format!(
"Error Server does not support range header: {}",
url
)));
continue;
}
if let Err(_e) = download::download_feedback_multi(&url, &into_file, rep.clone(), conn_count, Some(filesize)).await {
if let Err(_e) = download::download_feedback_multi(
&url,
&into_file,
rep.clone(),
conn_count,
Some(filesize),
)
.await
{
rep.send(DlStatus::DoneErr {
filename: file_name.to_string()
filename: file_name.to_string(),
});
}
};
}
}))
}
drop(tx);
dlreport::watch_and_print_reports(rx, cli_args.urls.len() as i32).await?;
dlreport::watch_and_print_reports(rx, raw_urls.len() as i32).await?;
join_all(joiners).await;