dnlmlr/ffdl
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

More refactor

Browse Source
This commit is contained in:
Daniel M 2022-03-31 01:16:39 +02:00
parent 2e10b54f32
commit 59de02d34d
5 changed files with 224 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
src/dlreport.rs
View File

@ -1,90 +1,82 @@
use std::collections::{ HashMap, VecDeque }; use std::collections::{HashMap, VecDeque};
use std::time::SystemTime;
use std::io::stdout; use std::io::stdout;
use std::time::SystemTime;
use tokio::sync::mpsc; use tokio::sync::mpsc;
use crossterm::cursor::{ MoveToPreviousLine }; use crossterm::cursor::MoveToPreviousLine;
use crossterm::execute; use crossterm::execute;
use crossterm::terminal::{ Clear, ClearType }; use crossterm::style::Print;
use crossterm::style::{ Print }; use crossterm::terminal::{Clear, ClearType};
use crate::errors::*; use crate::errors::*;
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub enum DlStatus { pub enum DlStatus {
Init { Init { bytes_total: u64, filename: String },
bytes_total: u64, Update { speed_mbps: f32, bytes_curr: u64 },
filename: String Done { duration_ms: u64 },
}, DoneErr { filename: String },
Update {
speed_mbps: f32,
bytes_curr: u64
},
Done {
duration_ms: u64
},
DoneErr {
filename: String
},
Skipped, Skipped,
Message(String) Message(String),
} }
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct DlReport { pub struct DlReport {
pub id: u32, pub id: u32,
pub status: DlStatus pub status: DlStatus,
} }
#[derive(Clone)] #[derive(Clone)]
pub struct DlReporter { pub struct DlReporter {
id: u32, id: u32,
transmitter: mpsc::UnboundedSender<DlReport> transmitter: mpsc::UnboundedSender<DlReport>,
} }
impl DlReporter { impl DlReporter {
pub fn new(id: u32, transmitter: mpsc::UnboundedSender<DlReport>) -> DlReporter { pub fn new(id: u32, transmitter: mpsc::UnboundedSender<DlReport>) -> DlReporter {
DlReporter { DlReporter {
id: id, id: id,
transmitter: transmitter transmitter: transmitter,
} }
} }
pub fn send(& self, status: DlStatus) { pub fn send(&self, status: DlStatus) {
// This should not fail, so unwrap it here instead propagating the error // This should not fail, so unwrap it here instead propagating the error
self.transmitter.send( self.transmitter
DlReport { .send(DlReport {
id: self.id, id: self.id,
status: status status,
} })
).unwrap(); .unwrap();
} }
} }
struct InfoHolder { struct InfoHolder {
filename: String, filename: String,
total_size: u64, total_size: u64,
progress: u64, progress: u64,
speed_mbps: f32 speed_mbps: f32,
} }
impl InfoHolder { impl InfoHolder {
fn new(filename: String, total_size: u64) -> InfoHolder { fn new(filename: String, total_size: u64) -> InfoHolder {
InfoHolder { InfoHolder {
filename, filename,
total_size, total_size,
progress: 0, progress: 0,
speed_mbps: 0.0 speed_mbps: 0.0,
} }
} }
} }
fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mut VecDeque<String>, moved_lines: u16, file_count_completed: i32, file_count_total: i32) -> ResBE<u16> { fn print_accumulated_report(
statuses: &HashMap<u32, InfoHolder>,
msg_queue: &mut VecDeque<String>,
moved_lines: u16,
file_count_completed: i32,
file_count_total: i32,
) -> ResBE<u16> {
let mut dl_speed_sum = 0.0; let mut dl_speed_sum = 0.0;
execute!( execute!(
@ -94,7 +86,6 @@ fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mu
)?; )?;
for msg in msg_queue.drain(..) { for msg in msg_queue.drain(..) {
let ct_now = chrono::Local::now(); let ct_now = chrono::Local::now();
execute!( execute!(
@ -113,12 +104,14 @@ fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mu
)?; )?;
for (_k, v) in statuses { for (_k, v) in statuses {
let percent_complete = v.progress as f64 / v.total_size as f64 * 100.0; let percent_complete = v.progress as f64 / v.total_size as f64 * 100.0;
execute!( execute!(
stdout(), stdout(),
Print(format!("Status: {:6.2} mb/s {:5.2}% completed '{}'", v.speed_mbps, percent_complete, v.filename)), Print(format!(
"Status: {:6.2} mb/s {:5.2}% completed '{}'",
v.speed_mbps, percent_complete, v.filename
)),
Clear(ClearType::UntilNewLine), Clear(ClearType::UntilNewLine),
Print("\n") Print("\n")
)?; )?;
@ -132,7 +125,10 @@ fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mu
stdout(), stdout(),
Clear(ClearType::CurrentLine), Clear(ClearType::CurrentLine),
Print("\n"), Print("\n"),
Print(format!(" =>> Accumulated download speed: {:6.2} mb/s {}/{} files, {:.0}%", dl_speed_sum, file_count_completed, file_count_total, file_percent_completed)), Print(format!(
" =>> Accumulated download speed: {:6.2} mb/s {}/{} files, {:.0}%",
dl_speed_sum, file_count_completed, file_count_total, file_percent_completed
)),
Clear(ClearType::UntilNewLine), Clear(ClearType::UntilNewLine),
Print("\n"), Print("\n"),
Clear(ClearType::FromCursorDown), Clear(ClearType::FromCursorDown),
@ -143,8 +139,10 @@ fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mu
Ok(statuses.len() as u16 + 3) Ok(statuses.len() as u16 + 3)
} }
pub async fn watch_and_print_reports(mut receiver: mpsc::UnboundedReceiver<DlReport>, file_count_total: i32) -> ResBE<()> { pub async fn watch_and_print_reports(
mut receiver: mpsc::UnboundedReceiver<DlReport>,
file_count_total: i32,
) -> ResBE<()> {
let mut statuses: HashMap<u32, InfoHolder> = HashMap::new(); let mut statuses: HashMap<u32, InfoHolder> = HashMap::new();
let mut moved_lines = 0; let mut moved_lines = 0;
let mut msg_queue = VecDeque::new(); let mut msg_queue = VecDeque::new();
@ -157,23 +155,25 @@ pub async fn watch_and_print_reports(mut receiver: mpsc::UnboundedReceiver<DlRep
while let Some(update) = receiver.recv().await { while let Some(update) = receiver.recv().await {
match update.status { match update.status {
DlStatus::Init { DlStatus::Init {
bytes_total, bytes_total,
filename filename,
} => { } => {
msg_queue.push_back(format!("Starting download for file '{}'", &filename)); msg_queue.push_back(format!("Starting download for file '{}'", &filename));
statuses.insert(update.id, InfoHolder::new(filename, bytes_total)); statuses.insert(update.id, InfoHolder::new(filename, bytes_total));
moved_lines = print_accumulated_report(&statuses, &mut msg_queue, moved_lines, file_count_done, file_count_total)?; moved_lines = print_accumulated_report(
&statuses,
}, &mut msg_queue,
moved_lines,
file_count_done,
file_count_total,
)?;
}
DlStatus::Update { DlStatus::Update {
speed_mbps, speed_mbps,
bytes_curr bytes_curr,
} => { } => {
// Scope the reference to prevent borrowing conflict later // Scope the reference to prevent borrowing conflict later
{ {
let s = &mut statuses.get_mut(&update.id).unwrap(); let s = &mut statuses.get_mut(&update.id).unwrap();
@ -182,16 +182,18 @@ pub async fn watch_and_print_reports(mut receiver: mpsc::UnboundedReceiver<DlRep
} }
if t_last.elapsed().unwrap().as_millis() > 500 { if t_last.elapsed().unwrap().as_millis() > 500 {
moved_lines = print_accumulated_report(&statuses, &mut msg_queue, moved_lines, file_count_done, file_count_total)?; moved_lines = print_accumulated_report(
&statuses,
&mut msg_queue,
moved_lines,
file_count_done,
file_count_total,
)?;
t_last = SystemTime::now(); t_last = SystemTime::now();
} }
}
}, DlStatus::Done { duration_ms } => {
DlStatus::Done {
duration_ms
} => {
msg_queue.push_back(format!( msg_queue.push_back(format!(
"Finished downloading '{}' with {:.2} mb in {:.2} seconds", "Finished downloading '{}' with {:.2} mb in {:.2} seconds",
&statuses.get(&update.id).unwrap().filename, &statuses.get(&update.id).unwrap().filename,
@ -203,47 +205,59 @@ pub async fn watch_and_print_reports(mut receiver: mpsc::UnboundedReceiver<DlRep
file_count_completed += 1; file_count_completed += 1;
file_count_done += 1; file_count_done += 1;
}
}, DlStatus::DoneErr { filename } => {
DlStatus::DoneErr { msg_queue.push_back(format!("Error: Download failed: '{}'", filename));
filename
} => {
msg_queue.push_back(format!(
"Error: Download failed: '{}'", filename
));
// Don't care if it exists, just make sure it is gone // Don't care if it exists, just make sure it is gone
statuses.remove(&update.id); statuses.remove(&update.id);
// Refresh display // Refresh display
moved_lines = print_accumulated_report(&statuses, &mut msg_queue, moved_lines, file_count_done, file_count_total)?; moved_lines = print_accumulated_report(
&statuses,
&mut msg_queue,
moved_lines,
file_count_done,
file_count_total,
)?;
t_last = SystemTime::now(); t_last = SystemTime::now();
file_count_failed += 1; file_count_failed += 1;
file_count_done += 1; file_count_done += 1;
}
},
DlStatus::Message(msg) => { DlStatus::Message(msg) => {
msg_queue.push_back(msg); msg_queue.push_back(msg);
moved_lines = print_accumulated_report(&statuses, &mut msg_queue, moved_lines, file_count_done, file_count_total)?; moved_lines = print_accumulated_report(
&statuses,
&mut msg_queue,
moved_lines,
file_count_done,
file_count_total,
)?;
t_last = SystemTime::now(); t_last = SystemTime::now();
}, }
DlStatus::Skipped => { DlStatus::Skipped => {
file_count_completed += 1; file_count_completed += 1;
file_count_done += 1; file_count_done += 1;
} }
} }
} }
print_accumulated_report(&statuses, &mut msg_queue, moved_lines, file_count_done, file_count_total)?; print_accumulated_report(
&statuses,
&mut msg_queue,
moved_lines,
file_count_done,
file_count_total,
)?;
execute!( execute!(
stdout(), stdout(),
MoveToPreviousLine(2), MoveToPreviousLine(2),
Print(format!("All done! {}/{} completed, {} failed\n", file_count_completed, file_count_total, file_count_failed)), Print(format!(
"All done! {}/{} completed, {} failed\n",
file_count_completed, file_count_total, file_count_failed
)),
Clear(ClearType::FromCursorDown) Clear(ClearType::FromCursorDown)
)?; )?;

211
src/download.rs
View File

@ -1,27 +1,25 @@
use std::path::Path;
use tokio::io::{ AsyncWriteExt, AsyncSeekExt };
use std::time::SystemTime;
use percent_encoding::percent_decode_str;
use std::io::SeekFrom;
use tokio::sync::mpsc;
use futures::stream::FuturesUnordered; use futures::stream::FuturesUnordered;
use futures::StreamExt; use futures::StreamExt;
use percent_encoding::percent_decode_str;
use std::io::SeekFrom;
use std::path::Path;
use std::time::SystemTime;
use tokio::io::{AsyncSeekExt, AsyncWriteExt};
use tokio::sync::mpsc;
use crate::errors::*;
use crate::dlreport::*; use crate::dlreport::*;
use crate::errors::*;
struct RollingAverage { struct RollingAverage {
index: usize, index: usize,
data: Vec<f64> data: Vec<f64>,
} }
impl RollingAverage { impl RollingAverage {
fn new(size: usize) -> Self { fn new(size: usize) -> Self {
RollingAverage { RollingAverage {
index: 0, index: 0,
data: Vec::with_capacity(size) data: Vec::with_capacity(size),
} }
} }
@ -51,38 +49,49 @@ impl RollingAverage {
fn add(&mut self, val: f64) { fn add(&mut self, val: f64) {
if self.data.capacity() == self.data.len() { if self.data.capacity() == self.data.len() {
self.data[self.index] = val; self.data[self.index] = val;
self.index += 1; self.index += 1;
if self.index >= self.data.capacity() { if self.index >= self.data.capacity() {
self.index = 0; self.index = 0;
} }
} else { } else {
self.data.push(val); self.data.push(val);
} }
} }
} }
/// Get the filename at the end of the given URL. This will decode the URL Encoding. /// Get the filename at the end of the given URL. This will decode the URL Encoding.
pub fn url_to_filename(url: &str) -> String { pub fn url_to_filename(url: &str) -> String {
let url_dec = percent_decode_str(&url).decode_utf8_lossy().to_owned().to_string(); let url_dec = percent_decode_str(&url)
let file_name = std::path::Path::new(&url_dec).file_name().unwrap().to_str().unwrap(); .decode_utf8_lossy()
.to_owned()
.to_string();
let file_name = std::path::Path::new(&url_dec)
.file_name()
.unwrap()
.to_str()
.unwrap();
// Split at ? and return the first part. If no ? is present, this just returns the full string // Split at ? and return the first part. If no ? is present, this just returns the full string
file_name.split("?").next().unwrap().to_string() file_name.split("?").next().unwrap().to_string()
} }
pub async fn download_feedback(url: &str, into_file: &str, rep: DlReporter, content_length: Option<u64>) -> ResBE<()> { pub async fn download_feedback(
url: &str,
download_feedback_chunks(url, into_file, rep, None, false, content_length).await into_file: &Path,
rep: DlReporter,
content_length: Option<u64>,
) -> ResBE<()> {
download_feedback_chunks(url, into_file, rep, None, content_length).await
} }
pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporter, from_to: Option<(u64, u64)>, seek_from: bool, content_length: Option<u64>) -> ResBE<()> { pub async fn download_feedback_chunks(
let into_file = Path::new(into_file); url: &str,
into_file: &Path,
rep: DlReporter,
from_to: Option<(u64, u64)>,
content_length: Option<u64>,
) -> ResBE<()> {
let mut content_length = match content_length { let mut content_length = match content_length {
Some(it) => it, Some(it) => it,
None => { None => {
@ -92,13 +101,12 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
}; };
// Send the HTTP request to download the given link // Send the HTTP request to download the given link
let mut req = reqwest::Client::new() let mut req = reqwest::Client::new().get(url);
.get(url);
// Add range header if needed // Add range header if needed
if let Some(from_to) = from_to { if let Some((from, to)) = from_to {
req = req.header(reqwest::header::RANGE, format!("bytes={}-{}", from_to.0, from_to.1)); req = req.header(reqwest::header::RANGE, format!("bytes={}-{}", from, to));
content_length = from_to.1 - from_to.0 + 1; content_length = to - from + 1;
} }
// Actually send the request and get the response // Actually send the request and get the response
@ -110,35 +118,25 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
} }
// Open the local output file // Open the local output file
let mut ofile = tokio::fs::OpenOptions::new(); let mut opts = tokio::fs::OpenOptions::new();
let mut ofile = opts
.create(true)
.write(true)
.truncate(!from_to.is_some())
.open(into_file)
.await?;
// Create the file if not existant if from_to.is_some() {
ofile.create(true)
// Open in write mode
.write(true);
// If seek_from is specified, the file cant be overwritten
if !seek_from {
// Delete and overwrite the file
ofile.truncate(true);
}
let mut ofile = ofile.open(into_file).await?;
if seek_from {
ofile.seek(SeekFrom::Start(from_to.unwrap().0)).await?; ofile.seek(SeekFrom::Start(from_to.unwrap().0)).await?;
} }
let filename = into_file.file_name().unwrap().to_str().unwrap(); let filename = into_file.file_name().unwrap().to_str().unwrap();
// Report the download start // Report the download start
rep.send( rep.send(DlStatus::Init {
DlStatus::Init {
bytes_total: content_length, bytes_total: content_length,
filename: filename.to_string() filename: filename.to_string(),
} });
);
let mut curr_progress = 0; let mut curr_progress = 0;
let mut speed_mbps = 0.0; let mut speed_mbps = 0.0;
@ -154,7 +152,6 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
// Read data from server as long as new data is available // Read data from server as long as new data is available
while let Some(chunk) = resp.chunk().await? { while let Some(chunk) = resp.chunk().await? {
let datalen = chunk.len() as u64; let datalen = chunk.len() as u64;
buff.extend(chunk); buff.extend(chunk);
@ -164,7 +161,6 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
// io bottleneck that occurs on HDDs with many small writes in different // io bottleneck that occurs on HDDs with many small writes in different
// files and offsets at the same time // files and offsets at the same time
if buff.len() >= 1_000_000 { if buff.len() >= 1_000_000 {
// Write the received data into the file // Write the received data into the file
ofile.write_all(&buff).await?; ofile.write_all(&buff).await?;
@ -179,14 +175,11 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
let t_elapsed = t_last_speed.elapsed()?.as_secs_f64(); let t_elapsed = t_last_speed.elapsed()?.as_secs_f64();
// Update the reported download speed after every 5MB or every second // Update the reported download speed after every 3MB or every second
// depending on what happens first // depending on what happens first
if last_bytecount >= 3_000_000 || t_elapsed >= 0.8 { if last_bytecount >= 3_000_000 || t_elapsed >= 0.8 {
// Update rolling average // Update rolling average
average_speed.add( average_speed.add(((last_bytecount as f64) / t_elapsed) / 1_000_000.0);
((last_bytecount as f64) / t_elapsed) / 1_000_000.0
);
speed_mbps = average_speed.value() as f32; speed_mbps = average_speed.value() as f32;
@ -196,12 +189,10 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
} }
// Send status update report // Send status update report
rep.send( rep.send(DlStatus::Update {
DlStatus::Update { speed_mbps,
speed_mbps: speed_mbps, bytes_curr: curr_progress,
bytes_curr: curr_progress });
}
);
} }
if buff.len() > 0 { if buff.len() > 0 {
@ -218,25 +209,23 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
let duration_ms = t_start.elapsed()?.as_millis() as u64; let duration_ms = t_start.elapsed()?.as_millis() as u64;
// Send report that the download is finished // Send report that the download is finished
rep.send( rep.send(DlStatus::Done { duration_ms });
DlStatus::Done {
duration_ms: duration_ms
}
);
Ok(()) Ok(())
} }
// This will spin up multiple tasks that and manage the status updates for them. // This will spin up multiple tasks that and manage the status updates for them.
// The combined status will be reported back to the caller // The combined status will be reported back to the caller
pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter, conn_count: u32, content_length: Option<u64>) -> ResBE<()> { pub async fn download_feedback_multi(
url: &str,
into_file: &Path,
rep: DlReporter,
conn_count: u32,
content_length: Option<u64>,
) -> ResBE<()> {
let content_length = match content_length { let content_length = match content_length {
Some(it) => it, Some(it) => it,
None => { None => http_get_filesize_and_range_support(url).await?.0,
let (content_length, _) = http_get_filesize_and_range_support(url).await?;
content_length
}
}; };
// Create zeroed file with 1 byte too much. This will be truncated on download // Create zeroed file with 1 byte too much. This will be truncated on download
@ -252,21 +241,16 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
let t_start = SystemTime::now(); let t_start = SystemTime::now();
for index in 0 .. conn_count { for index in 0..conn_count {
let url = url.clone().to_owned(); let url = url.clone().to_owned();
let into_file = into_file.clone().to_owned(); let into_file = into_file.clone().to_owned();
let tx = tx.clone(); let tx = tx.clone();
joiners.push(tokio::spawn(async move { joiners.push(tokio::spawn(async move {
let rep = DlReporter::new(index, tx.clone()); let rep = DlReporter::new(index, tx.clone());
let mut from_to = ( let mut from_to = (index as u64 * chunksize, (index + 1) as u64 * chunksize - 1);
index as u64 * chunksize,
(index+1) as u64 * chunksize - 1
);
if index == conn_count - 1 { if index == conn_count - 1 {
from_to.1 += rest; from_to.1 += rest;
@ -275,10 +259,17 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
let specific_content_length = from_to.1 - from_to.0 + 1; let specific_content_length = from_to.1 - from_to.0 + 1;
// Delay each chunk-download to reduce the number of simultanious connection attempts // Delay each chunk-download to reduce the number of simultanious connection attempts
tokio::time::sleep(tokio::time::Duration::from_millis(50 *index as u64)).await; tokio::time::sleep(tokio::time::Duration::from_millis(50 * index as u64)).await;
download_feedback_chunks(&url, &into_file, rep, Some(from_to), true, Some(specific_content_length)).await.map_err(|e| e.to_string())
download_feedback_chunks(
&url,
&into_file,
rep,
Some(from_to),
Some(specific_content_length),
)
.await
.map_err(|e| e.to_string())
})) }))
} }
@ -288,7 +279,7 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
rep.send(DlStatus::Init { rep.send(DlStatus::Init {
bytes_total: content_length, bytes_total: content_length,
filename: filename.to_string() filename: filename.to_string(),
}); });
let rep_task = rep.clone(); let rep_task = rep.clone();
@ -296,7 +287,6 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
let mut t_last = t_start.clone(); let mut t_last = t_start.clone();
let manager_handle = tokio::task::spawn(async move { let manager_handle = tokio::task::spawn(async move {
let rep = rep_task; let rep = rep_task;
//let mut dl_speeds = vec![0.0_f32; conn_count as usize]; //let mut dl_speeds = vec![0.0_f32; conn_count as usize];
let mut progresses = vec![0; conn_count as usize]; let mut progresses = vec![0; conn_count as usize];
@ -307,22 +297,17 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
while let Some(update) = rx.recv().await { while let Some(update) = rx.recv().await {
match update.status { match update.status {
DlStatus::Init { DlStatus::Init {
bytes_total: _, bytes_total: _,
filename: _ filename: _,
} => { } => {}
},
DlStatus::Update { DlStatus::Update {
speed_mbps: _, speed_mbps: _,
bytes_curr bytes_curr,
} => { } => {
//dl_speeds[update.id as usize] = speed_mbps; //dl_speeds[update.id as usize] = speed_mbps;
progresses[update.id as usize] = bytes_curr; progresses[update.id as usize] = bytes_curr;
let progress_curr = progresses.iter().sum(); let progress_curr = progresses.iter().sum();
let progress_delta = progress_curr - progress_last; let progress_delta = progress_curr - progress_last;
let t_elapsed = t_last.elapsed().unwrap().as_secs_f64(); let t_elapsed = t_last.elapsed().unwrap().as_secs_f64();
@ -331,48 +316,35 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
// currently executes always, but might change // currently executes always, but might change
if progress_delta >= 5_000_000 { if progress_delta >= 5_000_000 {
average_speed.add(((progress_delta as f64) / 1_000_000.0) / t_elapsed);
average_speed.add(
((progress_delta as f64) / 1_000_000.0) / t_elapsed
);
progress_last = progress_curr; progress_last = progress_curr;
t_last = SystemTime::now(); t_last = SystemTime::now();
} }
rep.send(DlStatus::Update { rep.send(DlStatus::Update {
speed_mbps: speed_mbps, speed_mbps: speed_mbps,
bytes_curr: progress_curr bytes_curr: progress_curr,
}); });
}
}, DlStatus::Done { duration_ms: _ } => {
DlStatus::Done {
duration_ms: _
} => {
//dl_speeds[update.id as usize] = 0.0; //dl_speeds[update.id as usize] = 0.0;
}
},
// Just forwared everything else to the calling receiver // Just forwared everything else to the calling receiver
_ => rep.send(update.status) _ => rep.send(update.status),
} }
} }
}); });
let mut joiners: FuturesUnordered<_> = joiners.into_iter().collect(); let mut joiners: FuturesUnordered<_> = joiners.into_iter().collect();
// Validate if the tasks were successful. This will always grab the next completed // Validate if the tasks were successful. This will always grab the next completed
// task, independent from the original order in the joiners list // task, independent from the original order in the joiners list
while let Some(output) = joiners.next().await { while let Some(output) = joiners.next().await {
// If any of the download tasks fail, abort the rest and delete the file // If any of the download tasks fail, abort the rest and delete the file
// since it is non-recoverable anyways // since it is non-recoverable anyways
if let Err(e) = output? { if let Err(e) = output? {
for handle in joiners.iter() { for handle in joiners.iter() {
handle.abort(); handle.abort();
} }
@ -398,14 +370,13 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
ofile.set_len(content_length).await?; ofile.set_len(content_length).await?;
rep.send(DlStatus::Done { rep.send(DlStatus::Done {
duration_ms: t_start.elapsed()?.as_millis() as u64 duration_ms: t_start.elapsed()?.as_millis() as u64,
}); });
Ok(()) Ok(())
} }
async fn create_zeroed_file(file: &str, filesize: usize) -> ResBE<()> { async fn create_zeroed_file(file: &Path, filesize: usize) -> ResBE<()> {
let ofile = tokio::fs::OpenOptions::new() let ofile = tokio::fs::OpenOptions::new()
.create(true) .create(true)
// Open in write mode // Open in write mode
@ -421,14 +392,11 @@ async fn create_zeroed_file(file: &str, filesize: usize) -> ResBE<()> {
} }
pub async fn http_get_filesize_and_range_support(url: &str) -> ResBE<(u64, bool)> { pub async fn http_get_filesize_and_range_support(url: &str) -> ResBE<(u64, bool)> {
let resp = reqwest::Client::new() let resp = reqwest::Client::new().head(url).send().await?;
.head(url)
.send().await?;
if let Some(filesize) = resp.headers().get(reqwest::header::CONTENT_LENGTH) { if let Some(filesize) = resp.headers().get(reqwest::header::CONTENT_LENGTH) {
if let Ok(val_str) = filesize.to_str() { if let Ok(val_str) = filesize.to_str() {
if let Ok(val) = val_str.parse::<u64>() { if let Ok(val) = val_str.parse::<u64>() {
let mut range_supported = false; let mut range_supported = false;
if let Some(range) = resp.headers().get(reqwest::header::ACCEPT_RANGES) { if let Some(range) = resp.headers().get(reqwest::header::ACCEPT_RANGES) {
@ -447,9 +415,6 @@ pub async fn http_get_filesize_and_range_support(url: &str) -> ResBE<(u64, bool)
Err(DlError::ContentLengthUnknown.into()) Err(DlError::ContentLengthUnknown.into())
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
@ -496,7 +461,6 @@ mod tests {
assert_eq!(20.0, ra.data[1]); assert_eq!(20.0, ra.data[1]);
assert_eq!(30.0, ra.data[2]); assert_eq!(30.0, ra.data[2]);
ra.add(50.0); ra.add(50.0);
ra.add(60.0); ra.add(60.0);
ra.add(70.0); ra.add(70.0);
@ -504,6 +468,5 @@ mod tests {
assert_eq!(70.0, ra.data[0]); assert_eq!(70.0, ra.data[0]);
assert_eq!(50.0, ra.data[1]); assert_eq!(50.0, ra.data[1]);
assert_eq!(60.0, ra.data[2]); assert_eq!(60.0, ra.data[2]);
} }
} }

8
src/errors.rs
View File

@ -1,5 +1,5 @@
use std::error::Error; use std::error::Error;
use std::fmt::{ self, Display, Formatter }; use std::fmt::{self, Display, Formatter};
/// Result Boxed Error /// Result Boxed Error
pub type ResBE<T> = Result<T, Box<dyn Error>>; pub type ResBE<T> = Result<T, Box<dyn Error>>;
@ -10,20 +10,18 @@ pub enum DlError {
BadHttpStatus, BadHttpStatus,
ContentLengthUnknown, ContentLengthUnknown,
HttpNoData, HttpNoData,
Other(String) Other(String),
} }
impl Error for DlError {} impl Error for DlError {}
impl Display for DlError { impl Display for DlError {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match self { match self {
DlError::BadHttpStatus => write!(f, "Bad http response status"), DlError::BadHttpStatus => write!(f, "Bad http response status"),
DlError::ContentLengthUnknown => write!(f, "Content-Length is unknown"), DlError::ContentLengthUnknown => write!(f, "Content-Length is unknown"),
DlError::HttpNoData => write!(f, "Http server sent no more data"), DlError::HttpNoData => write!(f, "Http server sent no more data"),
DlError::Other(s) => write!(f, "Unknown download error: '{}'", s) DlError::Other(s) => write!(f, "Unknown download error: '{}'", s),
} }
} }
} }

18
src/main.rs
View File

@ -1,4 +1,10 @@
use std::{collections::VecDeque, path::Path, process::exit, sync::Arc, time::SystemTime}; use std::{
collections::VecDeque,
path::{Path, PathBuf},
process::exit,
sync::Arc,
time::SystemTime,
};
use clap::Parser; use clap::Parser;
use download::{download_feedback, download_feedback_multi, http_get_filesize_and_range_support}; use download::{download_feedback, download_feedback_multi, http_get_filesize_and_range_support};
@ -137,13 +143,13 @@ async fn download_job(urls: SyncQueue, reporter: UnboundedSender<DlReport>, cli_
.clone() .clone()
.unwrap_or_else(|| download::url_to_filename(&url).into()); .unwrap_or_else(|| download::url_to_filename(&url).into());
let into_file = cli_args let into_file: PathBuf = cli_args
.outdir .outdir
.join(Path::new(&file_name)) .join(Path::new(&file_name))
.to_str() .to_str()
.unwrap() .unwrap()
.to_string(); .to_string()
let path_into_file = Path::new(&into_file); .into();
let (filesize, range_supported) = match http_get_filesize_and_range_support(&url).await { let (filesize, range_supported) = match http_get_filesize_and_range_support(&url).await {
Ok((filesize, range_supported)) => (filesize, range_supported), Ok((filesize, range_supported)) => (filesize, range_supported),
@ -157,8 +163,8 @@ async fn download_job(urls: SyncQueue, reporter: UnboundedSender<DlReport>, cli_
}; };
// If file with same name is present locally, check filesize // If file with same name is present locally, check filesize
if path_into_file.exists() { if into_file.exists() {
let local_filesize = std::fs::metadata(path_into_file).unwrap().len(); let local_filesize = std::fs::metadata(&into_file).unwrap().len();
if filesize == local_filesize { if filesize == local_filesize {
reporter.send(DlStatus::Message(format!( reporter.send(DlStatus::Message(format!(

12
src/zippy.rs
View File

@ -1,5 +1,5 @@
use regex::Regex; use regex::Regex;
use std::io::{ Error, ErrorKind }; use std::io::{Error, ErrorKind};
use crate::errors::ResBE; use crate::errors::ResBE;
@ -18,7 +18,6 @@ document.getElementById('dlbutton').href = "/d/0Ky7p1C6/" + (186549 % 51245 + 18
``` ```
*/ */
pub async fn resolve_link(url: &str) -> ResBE<String> { pub async fn resolve_link(url: &str) -> ResBE<String> {
// Regex to check if the provided url is a zippyshare download url // Regex to check if the provided url is a zippyshare download url
let re = Regex::new(r"(https://www\d*\.zippyshare\.com)")?; let re = Regex::new(r"(https://www\d*\.zippyshare\.com)")?;
if !re.is_match(&url) { if !re.is_match(&url) {
@ -29,15 +28,16 @@ pub async fn resolve_link(url: &str) -> ResBE<String> {
let base_host = &re.captures(&url).unwrap()[0]; let base_host = &re.captures(&url).unwrap()[0];
// Download the html body for the download page // Download the html body for the download page
let body = reqwest::get(url).await? let body = reqwest::get(url).await?.text().await?;
.text().await?;
// Regex to match the javascript part of the html that generates the real download link // Regex to match the javascript part of the html that generates the real download link
let re_link = Regex::new(r#"document\.getElementById\('dlbutton'\)\.href = "(/d/.+/)" \+ \((\d+) % (\d+) \+ \d+ % (\d+)\) \+ "(.+)";"#)?; let re_link = Regex::new(
r#"document\.getElementById\('dlbutton'\)\.href = "(/d/.+/)" \+ \((\d+) % (\d+) \+ \d+ % (\d+)\) \+ "(.+)";"#,
)?;
let cap_link = match re_link.captures(&body) { let cap_link = match re_link.captures(&body) {
Some(cap) => cap, Some(cap) => cap,
None => return Err(Error::new(ErrorKind::Other, "Link not found").into()) None => return Err(Error::new(ErrorKind::Other, "Link not found").into()),
}; };
let url_start = &cap_link[1]; let url_start = &cap_link[1];