dnlmlr/ffdl
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

More refactor

Browse Source
This commit is contained in:
Daniel M 2022-03-31 01:16:39 +02:00
parent 2e10b54f32
commit 59de02d34d
5 changed files with 224 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

184
src/dlreport.rs
View File

@ -1,90 +1,82 @@
use std::collections::{ HashMap, VecDeque };
use std::time::SystemTime;
use std::collections::{HashMap, VecDeque};
use std::io::stdout;
use std::time::SystemTime;
use tokio::sync::mpsc;
use crossterm::cursor::{ MoveToPreviousLine };
use crossterm::cursor::MoveToPreviousLine;
use crossterm::execute;
use crossterm::terminal::{ Clear, ClearType };
use crossterm::style::{ Print };
use crossterm::style::Print;
use crossterm::terminal::{Clear, ClearType};
use crate::errors::*;
#[derive(Clone, Debug)]
pub enum DlStatus {
Init {
bytes_total: u64,
filename: String
},
Update {
speed_mbps: f32,
bytes_curr: u64
},
Done {
duration_ms: u64
},
DoneErr {
filename: String
},
Init { bytes_total: u64, filename: String },
Update { speed_mbps: f32, bytes_curr: u64 },
Done { duration_ms: u64 },
DoneErr { filename: String },
Skipped,
Message(String)
Message(String),
}
#[derive(Clone, Debug)]
pub struct DlReport {
pub id: u32,
pub status: DlStatus
pub status: DlStatus,
}
#[derive(Clone)]
pub struct DlReporter {
id: u32,
transmitter: mpsc::UnboundedSender<DlReport>
transmitter: mpsc::UnboundedSender<DlReport>,
}
impl DlReporter {
pub fn new(id: u32, transmitter: mpsc::UnboundedSender<DlReport>) -> DlReporter {
DlReporter {
id: id,
transmitter: transmitter
transmitter: transmitter,
}
}
pub fn send(& self, status: DlStatus) {
pub fn send(&self, status: DlStatus) {
// This should not fail, so unwrap it here instead propagating the error
self.transmitter.send(
DlReport {
self.transmitter
.send(DlReport {
id: self.id,
status: status
}
).unwrap();
status,
})
.unwrap();
}
}
struct InfoHolder {
filename: String,
total_size: u64,
progress: u64,
speed_mbps: f32
filename: String,
total_size: u64,
progress: u64,
speed_mbps: f32,
}
impl InfoHolder {
fn new(filename: String, total_size: u64) -> InfoHolder {
InfoHolder {
filename,
total_size,
progress: 0,
speed_mbps: 0.0
speed_mbps: 0.0,
}
}
}
fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mut VecDeque<String>, moved_lines: u16, file_count_completed: i32, file_count_total: i32) -> ResBE<u16> {
fn print_accumulated_report(
statuses: &HashMap<u32, InfoHolder>,
msg_queue: &mut VecDeque<String>,
moved_lines: u16,
file_count_completed: i32,
file_count_total: i32,
) -> ResBE<u16> {
let mut dl_speed_sum = 0.0;
execute!(
@ -94,7 +86,6 @@ fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mu
)?;
for msg in msg_queue.drain(..) {
let ct_now = chrono::Local::now();
execute!(
@ -104,7 +95,7 @@ fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mu
Print("\n")
)?;
}
execute!(
stdout(),
Print(format!("----------------------------------------")),
@ -113,12 +104,14 @@ fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mu
)?;
for (_k, v) in statuses {
let percent_complete = v.progress as f64 / v.total_size as f64 * 100.0;
execute!(
stdout(),
Print(format!("Status: {:6.2} mb/s {:5.2}% completed '{}'", v.speed_mbps, percent_complete, v.filename)),
Print(format!(
"Status: {:6.2} mb/s {:5.2}% completed '{}'",
v.speed_mbps, percent_complete, v.filename
)),
Clear(ClearType::UntilNewLine),
Print("\n")
)?;
@ -132,7 +125,10 @@ fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mu
stdout(),
Clear(ClearType::CurrentLine),
Print("\n"),
Print(format!(" =>> Accumulated download speed: {:6.2} mb/s {}/{} files, {:.0}%", dl_speed_sum, file_count_completed, file_count_total, file_percent_completed)),
Print(format!(
" =>> Accumulated download speed: {:6.2} mb/s {}/{} files, {:.0}%",
dl_speed_sum, file_count_completed, file_count_total, file_percent_completed
)),
Clear(ClearType::UntilNewLine),
Print("\n"),
Clear(ClearType::FromCursorDown),
@ -143,8 +139,10 @@ fn print_accumulated_report(statuses: & HashMap<u32, InfoHolder>, msg_queue: &mu
Ok(statuses.len() as u16 + 3)
}
pub async fn watch_and_print_reports(mut receiver: mpsc::UnboundedReceiver<DlReport>, file_count_total: i32) -> ResBE<()> {
pub async fn watch_and_print_reports(
mut receiver: mpsc::UnboundedReceiver<DlReport>,
file_count_total: i32,
) -> ResBE<()> {
let mut statuses: HashMap<u32, InfoHolder> = HashMap::new();
let mut moved_lines = 0;
let mut msg_queue = VecDeque::new();
@ -157,23 +155,25 @@ pub async fn watch_and_print_reports(mut receiver: mpsc::UnboundedReceiver<DlRep
while let Some(update) = receiver.recv().await {
match update.status {
DlStatus::Init {
bytes_total,
filename
filename,
} => {
msg_queue.push_back(format!("Starting download for file '{}'", &filename));
statuses.insert(update.id, InfoHolder::new(filename, bytes_total));
moved_lines = print_accumulated_report(&statuses, &mut msg_queue, moved_lines, file_count_done, file_count_total)?;
},
moved_lines = print_accumulated_report(
&statuses,
&mut msg_queue,
moved_lines,
file_count_done,
file_count_total,
)?;
}
DlStatus::Update {
speed_mbps,
bytes_curr
bytes_curr,
} => {
// Scope the reference to prevent borrowing conflict later
{
let s = &mut statuses.get_mut(&update.id).unwrap();
@ -182,68 +182,82 @@ pub async fn watch_and_print_reports(mut receiver: mpsc::UnboundedReceiver<DlRep
}
if t_last.elapsed().unwrap().as_millis() > 500 {
moved_lines = print_accumulated_report(&statuses, &mut msg_queue, moved_lines, file_count_done, file_count_total)?;
moved_lines = print_accumulated_report(
&statuses,
&mut msg_queue,
moved_lines,
file_count_done,
file_count_total,
)?;
t_last = SystemTime::now();
}
},
DlStatus::Done {
duration_ms
} => {
}
DlStatus::Done { duration_ms } => {
msg_queue.push_back(format!(
"Finished downloading '{}' with {:.2} mb in {:.2} seconds",
&statuses.get(&update.id).unwrap().filename,
(statuses.get(&update.id).unwrap().total_size as f32 / 1_000_000.0),
(duration_ms as f32 / 1_000.0)
"Finished downloading '{}' with {:.2} mb in {:.2} seconds",
&statuses.get(&update.id).unwrap().filename,
(statuses.get(&update.id).unwrap().total_size as f32 / 1_000_000.0),
(duration_ms as f32 / 1_000.0)
));
statuses.remove(&update.id);
file_count_completed += 1;
file_count_done += 1;
},
DlStatus::DoneErr {
filename
} => {
msg_queue.push_back(format!(
"Error: Download failed: '{}'", filename
));
}
DlStatus::DoneErr { filename } => {
msg_queue.push_back(format!("Error: Download failed: '{}'", filename));
// Don't care if it exists, just make sure it is gone
statuses.remove(&update.id);
// Refresh display
moved_lines = print_accumulated_report(&statuses, &mut msg_queue, moved_lines, file_count_done, file_count_total)?;
moved_lines = print_accumulated_report(
&statuses,
&mut msg_queue,
moved_lines,
file_count_done,
file_count_total,
)?;
t_last = SystemTime::now();
file_count_failed += 1;
file_count_done += 1;
},
}
DlStatus::Message(msg) => {
msg_queue.push_back(msg);
moved_lines = print_accumulated_report(&statuses, &mut msg_queue, moved_lines, file_count_done, file_count_total)?;
moved_lines = print_accumulated_report(
&statuses,
&mut msg_queue,
moved_lines,
file_count_done,
file_count_total,
)?;
t_last = SystemTime::now();
},
}
DlStatus::Skipped => {
file_count_completed += 1;
file_count_done += 1;
}
}
}
print_accumulated_report(&statuses, &mut msg_queue, moved_lines, file_count_done, file_count_total)?;
print_accumulated_report(
&statuses,
&mut msg_queue,
moved_lines,
file_count_done,
file_count_total,
)?;
execute!(
stdout(),
MoveToPreviousLine(2),
Print(format!("All done! {}/{} completed, {} failed\n", file_count_completed, file_count_total, file_count_failed)),
Print(format!(
"All done! {}/{} completed, {} failed\n",
file_count_completed, file_count_total, file_count_failed
)),
Clear(ClearType::FromCursorDown)
)?;

239
src/download.rs
View File

@ -1,27 +1,25 @@
use std::path::Path;
use tokio::io::{ AsyncWriteExt, AsyncSeekExt };
use std::time::SystemTime;
use percent_encoding::percent_decode_str;
use std::io::SeekFrom;
use tokio::sync::mpsc;
use futures::stream::FuturesUnordered;
use futures::StreamExt;
use percent_encoding::percent_decode_str;
use std::io::SeekFrom;
use std::path::Path;
use std::time::SystemTime;
use tokio::io::{AsyncSeekExt, AsyncWriteExt};
use tokio::sync::mpsc;
use crate::errors::*;
use crate::dlreport::*;
use crate::errors::*;
struct RollingAverage {
index: usize,
data: Vec<f64>
data: Vec<f64>,
}
impl RollingAverage {
fn new(size: usize) -> Self {
RollingAverage {
index: 0,
data: Vec::with_capacity(size)
data: Vec::with_capacity(size),
}
}
@ -51,38 +49,49 @@ impl RollingAverage {
fn add(&mut self, val: f64) {
if self.data.capacity() == self.data.len() {
self.data[self.index] = val;
self.index += 1;
if self.index >= self.data.capacity() {
self.index = 0;
}
} else {
self.data.push(val);
}
}
}
/// Get the filename at the end of the given URL. This will decode the URL Encoding.
pub fn url_to_filename(url: &str) -> String {
let url_dec = percent_decode_str(&url).decode_utf8_lossy().to_owned().to_string();
let file_name = std::path::Path::new(&url_dec).file_name().unwrap().to_str().unwrap();
let url_dec = percent_decode_str(&url)
.decode_utf8_lossy()
.to_owned()
.to_string();
let file_name = std::path::Path::new(&url_dec)
.file_name()
.unwrap()
.to_str()
.unwrap();
// Split at ? and return the first part. If no ? is present, this just returns the full string
file_name.split("?").next().unwrap().to_string()
}
pub async fn download_feedback(url: &str, into_file: &str, rep: DlReporter, content_length: Option<u64>) -> ResBE<()> {
download_feedback_chunks(url, into_file, rep, None, false, content_length).await
pub async fn download_feedback(
url: &str,
into_file: &Path,
rep: DlReporter,
content_length: Option<u64>,
) -> ResBE<()> {
download_feedback_chunks(url, into_file, rep, None, content_length).await
}
pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporter, from_to: Option<(u64, u64)>, seek_from: bool, content_length: Option<u64>) -> ResBE<()> {
let into_file = Path::new(into_file);
pub async fn download_feedback_chunks(
url: &str,
into_file: &Path,
rep: DlReporter,
from_to: Option<(u64, u64)>,
content_length: Option<u64>,
) -> ResBE<()> {
let mut content_length = match content_length {
Some(it) => it,
None => {
@ -92,15 +101,14 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
};
// Send the HTTP request to download the given link
let mut req = reqwest::Client::new()
.get(url);
let mut req = reqwest::Client::new().get(url);
// Add range header if needed
if let Some(from_to) = from_to {
req = req.header(reqwest::header::RANGE, format!("bytes={}-{}", from_to.0, from_to.1));
content_length = from_to.1 - from_to.0 + 1;
if let Some((from, to)) = from_to {
req = req.header(reqwest::header::RANGE, format!("bytes={}-{}", from, to));
content_length = to - from + 1;
}
// Actually send the request and get the response
let mut resp = req.send().await?;
@ -108,37 +116,27 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
if !resp.status().is_success() {
return Err(DlError::BadHttpStatus.into());
}
// Open the local output file
let mut ofile = tokio::fs::OpenOptions::new();
// Create the file if not existant
ofile.create(true)
// Open in write mode
.write(true);
let mut opts = tokio::fs::OpenOptions::new();
let mut ofile = opts
.create(true)
.write(true)
.truncate(!from_to.is_some())
.open(into_file)
.await?;
// If seek_from is specified, the file cant be overwritten
if !seek_from {
// Delete and overwrite the file
ofile.truncate(true);
}
let mut ofile = ofile.open(into_file).await?;
if seek_from {
if from_to.is_some() {
ofile.seek(SeekFrom::Start(from_to.unwrap().0)).await?;
}
let filename = into_file.file_name().unwrap().to_str().unwrap();
// Report the download start
rep.send(
DlStatus::Init {
bytes_total: content_length,
filename: filename.to_string()
}
);
rep.send(DlStatus::Init {
bytes_total: content_length,
filename: filename.to_string(),
});
let mut curr_progress = 0;
let mut speed_mbps = 0.0;
@ -154,17 +152,15 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
// Read data from server as long as new data is available
while let Some(chunk) = resp.chunk().await? {
let datalen = chunk.len() as u64;
buff.extend(chunk);
// Buffer in memory first and only write to disk if the threshold is reached.
// This reduces the number of small disk writes and thereby reduces the
// This reduces the number of small disk writes and thereby reduces the
// io bottleneck that occurs on HDDs with many small writes in different
// files and offsets at the same time
if buff.len() >= 1_000_000 {
// Write the received data into the file
ofile.write_all(&buff).await?;
@ -179,14 +175,11 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
let t_elapsed = t_last_speed.elapsed()?.as_secs_f64();
// Update the reported download speed after every 5MB or every second
// Update the reported download speed after every 3MB or every second
// depending on what happens first
if last_bytecount >= 3_000_000 || t_elapsed >= 0.8 {
// Update rolling average
average_speed.add(
((last_bytecount as f64) / t_elapsed) / 1_000_000.0
);
average_speed.add(((last_bytecount as f64) / t_elapsed) / 1_000_000.0);
speed_mbps = average_speed.value() as f32;
@ -196,12 +189,10 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
}
// Send status update report
rep.send(
DlStatus::Update {
speed_mbps: speed_mbps,
bytes_curr: curr_progress
}
);
rep.send(DlStatus::Update {
speed_mbps,
bytes_curr: curr_progress,
});
}
if buff.len() > 0 {
@ -211,34 +202,32 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
if curr_progress != content_length {
return Err(DlError::HttpNoData.into());
}
// Ensure that IO is completed
//ofile.flush().await?;
let duration_ms = t_start.elapsed()?.as_millis() as u64;
// Send report that the download is finished
rep.send(
DlStatus::Done {
duration_ms: duration_ms
}
);
rep.send(DlStatus::Done { duration_ms });
Ok(())
}
// This will spin up multiple tasks that and manage the status updates for them.
// The combined status will be reported back to the caller
pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter, conn_count: u32, content_length: Option<u64>) -> ResBE<()> {
pub async fn download_feedback_multi(
url: &str,
into_file: &Path,
rep: DlReporter,
conn_count: u32,
content_length: Option<u64>,
) -> ResBE<()> {
let content_length = match content_length {
Some(it) => it,
None => {
let (content_length, _) = http_get_filesize_and_range_support(url).await?;
content_length
}
None => http_get_filesize_and_range_support(url).await?.0,
};
// Create zeroed file with 1 byte too much. This will be truncated on download
// completion and can indicate that the file is not suitable for continuation
create_zeroed_file(into_file, content_length as usize + 1).await?;
@ -252,21 +241,16 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
let t_start = SystemTime::now();
for index in 0 .. conn_count {
for index in 0..conn_count {
let url = url.clone().to_owned();
let into_file = into_file.clone().to_owned();
let tx = tx.clone();
joiners.push(tokio::spawn(async move {
let rep = DlReporter::new(index, tx.clone());
let mut from_to = (
index as u64 * chunksize,
(index+1) as u64 * chunksize - 1
);
let mut from_to = (index as u64 * chunksize, (index + 1) as u64 * chunksize - 1);
if index == conn_count - 1 {
from_to.1 += rest;
@ -275,10 +259,17 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
let specific_content_length = from_to.1 - from_to.0 + 1;
// Delay each chunk-download to reduce the number of simultanious connection attempts
tokio::time::sleep(tokio::time::Duration::from_millis(50 *index as u64)).await;
download_feedback_chunks(&url, &into_file, rep, Some(from_to), true, Some(specific_content_length)).await.map_err(|e| e.to_string())
tokio::time::sleep(tokio::time::Duration::from_millis(50 * index as u64)).await;
download_feedback_chunks(
&url,
&into_file,
rep,
Some(from_to),
Some(specific_content_length),
)
.await
.map_err(|e| e.to_string())
}))
}
@ -288,7 +279,7 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
rep.send(DlStatus::Init {
bytes_total: content_length,
filename: filename.to_string()
filename: filename.to_string(),
});
let rep_task = rep.clone();
@ -296,7 +287,6 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
let mut t_last = t_start.clone();
let manager_handle = tokio::task::spawn(async move {
let rep = rep_task;
//let mut dl_speeds = vec![0.0_f32; conn_count as usize];
let mut progresses = vec![0; conn_count as usize];
@ -307,22 +297,17 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
while let Some(update) = rx.recv().await {
match update.status {
DlStatus::Init {
bytes_total: _,
filename: _
} => {
},
filename: _,
} => {}
DlStatus::Update {
speed_mbps: _,
bytes_curr
bytes_curr,
} => {
//dl_speeds[update.id as usize] = speed_mbps;
progresses[update.id as usize] = bytes_curr;
let progress_curr = progresses.iter().sum();
let progress_delta = progress_curr - progress_last;
let t_elapsed = t_last.elapsed().unwrap().as_secs_f64();
@ -331,48 +316,35 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
// currently executes always, but might change
if progress_delta >= 5_000_000 {
average_speed.add(
((progress_delta as f64) / 1_000_000.0) / t_elapsed
);
average_speed.add(((progress_delta as f64) / 1_000_000.0) / t_elapsed);
progress_last = progress_curr;
t_last = SystemTime::now();
}
rep.send(DlStatus::Update {
speed_mbps: speed_mbps,
bytes_curr: progress_curr
bytes_curr: progress_curr,
});
},
DlStatus::Done {
duration_ms: _
} => {
}
DlStatus::Done { duration_ms: _ } => {
//dl_speeds[update.id as usize] = 0.0;
},
// Just forwared everything else to the calling receiver
_ => rep.send(update.status)
}
// Just forwared everything else to the calling receiver
_ => rep.send(update.status),
}
}
});
let mut joiners: FuturesUnordered<_> = joiners.into_iter().collect();
// Validate if the tasks were successful. This will always grab the next completed
// Validate if the tasks were successful. This will always grab the next completed
// task, independent from the original order in the joiners list
while let Some(output) = joiners.next().await {
// If any of the download tasks fail, abort the rest and delete the file
// since it is non-recoverable anyways
if let Err(e) = output? {
for handle in joiners.iter() {
handle.abort();
}
@ -398,14 +370,13 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
ofile.set_len(content_length).await?;
rep.send(DlStatus::Done {
duration_ms: t_start.elapsed()?.as_millis() as u64
duration_ms: t_start.elapsed()?.as_millis() as u64,
});
Ok(())
}
async fn create_zeroed_file(file: &str, filesize: usize) -> ResBE<()> {
async fn create_zeroed_file(file: &Path, filesize: usize) -> ResBE<()> {
let ofile = tokio::fs::OpenOptions::new()
.create(true)
// Open in write mode
@ -414,21 +385,18 @@ async fn create_zeroed_file(file: &str, filesize: usize) -> ResBE<()> {
.truncate(true)
.open(file)
.await?;
ofile.set_len(filesize as u64).await?;
Ok(())
}
pub async fn http_get_filesize_and_range_support(url: &str) -> ResBE<(u64, bool)> {
let resp = reqwest::Client::new()
.head(url)
.send().await?;
let resp = reqwest::Client::new().head(url).send().await?;
if let Some(filesize) = resp.headers().get(reqwest::header::CONTENT_LENGTH) {
if let Ok(val_str) = filesize.to_str() {
if let Ok(val) = val_str.parse::<u64>() {
let mut range_supported = false;
if let Some(range) = resp.headers().get(reqwest::header::ACCEPT_RANGES) {
@ -447,9 +415,6 @@ pub async fn http_get_filesize_and_range_support(url: &str) -> ResBE<(u64, bool)
Err(DlError::ContentLengthUnknown.into())
}
#[cfg(test)]
mod tests {
@ -485,7 +450,7 @@ mod tests {
// This should replace the oldest value (index 1)
ra.add(40.0);
assert_eq!(3, ra.data.len());
assert_eq!(3, ra.data.capacity());
@ -496,7 +461,6 @@ mod tests {
assert_eq!(20.0, ra.data[1]);
assert_eq!(30.0, ra.data[2]);
ra.add(50.0);
ra.add(60.0);
ra.add(70.0);
@ -504,6 +468,5 @@ mod tests {
assert_eq!(70.0, ra.data[0]);
assert_eq!(50.0, ra.data[1]);
assert_eq!(60.0, ra.data[2]);
}
}

10
src/errors.rs
View File

@ -1,5 +1,5 @@
use std::error::Error;
use std::fmt::{ self, Display, Formatter };
use std::fmt::{self, Display, Formatter};
/// Result Boxed Error
pub type ResBE<T> = Result<T, Box<dyn Error>>;
@ -10,20 +10,18 @@ pub enum DlError {
BadHttpStatus,
ContentLengthUnknown,
HttpNoData,
Other(String)
Other(String),
}
impl Error for DlError {}
impl Display for DlError {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match self {
DlError::BadHttpStatus => write!(f, "Bad http response status"),
DlError::ContentLengthUnknown => write!(f, "Content-Length is unknown"),
DlError::HttpNoData => write!(f, "Http server sent no more data"),
DlError::Other(s) => write!(f, "Unknown download error: '{}'", s)
DlError::Other(s) => write!(f, "Unknown download error: '{}'", s),
}
}
}
}

18
src/main.rs
View File

@ -1,4 +1,10 @@
use std::{collections::VecDeque, path::Path, process::exit, sync::Arc, time::SystemTime};
use std::{
collections::VecDeque,
path::{Path, PathBuf},
process::exit,
sync::Arc,
time::SystemTime,
};
use clap::Parser;
use download::{download_feedback, download_feedback_multi, http_get_filesize_and_range_support};
@ -137,13 +143,13 @@ async fn download_job(urls: SyncQueue, reporter: UnboundedSender<DlReport>, cli_
.clone()
.unwrap_or_else(|| download::url_to_filename(&url).into());
let into_file = cli_args
let into_file: PathBuf = cli_args
.outdir
.join(Path::new(&file_name))
.to_str()
.unwrap()
.to_string();
let path_into_file = Path::new(&into_file);
.to_string()
.into();
let (filesize, range_supported) = match http_get_filesize_and_range_support(&url).await {
Ok((filesize, range_supported)) => (filesize, range_supported),
@ -157,8 +163,8 @@ async fn download_job(urls: SyncQueue, reporter: UnboundedSender<DlReport>, cli_
};
// If file with same name is present locally, check filesize
if path_into_file.exists() {
let local_filesize = std::fs::metadata(path_into_file).unwrap().len();
if into_file.exists() {
let local_filesize = std::fs::metadata(&into_file).unwrap().len();
if filesize == local_filesize {
reporter.send(DlStatus::Message(format!(

16
src/zippy.rs
View File

@ -1,5 +1,5 @@
use regex::Regex;
use std::io::{ Error, ErrorKind };
use std::io::{Error, ErrorKind};
use crate::errors::ResBE;
@ -13,12 +13,11 @@ Link generation code:
- `$6` is dependent on the file
- The numbers in the calculation part ($2`, `$3`, `$4` and `$5`) are hard coded
```
```
document.getElementById('dlbutton').href = "/d/0Ky7p1C6/" + (186549 % 51245 + 186549 % 913) + "/some-file-name.part1.rar";
```
*/
pub async fn resolve_link(url: &str) -> ResBE<String> {
// Regex to check if the provided url is a zippyshare download url
let re = Regex::new(r"(https://www\d*\.zippyshare\.com)")?;
if !re.is_match(&url) {
@ -29,17 +28,18 @@ pub async fn resolve_link(url: &str) -> ResBE<String> {
let base_host = &re.captures(&url).unwrap()[0];
// Download the html body for the download page
let body = reqwest::get(url).await?
.text().await?;
let body = reqwest::get(url).await?.text().await?;
// Regex to match the javascript part of the html that generates the real download link
let re_link = Regex::new(r#"document\.getElementById\('dlbutton'\)\.href = "(/d/.+/)" \+ \((\d+) % (\d+) \+ \d+ % (\d+)\) \+ "(.+)";"#)?;
let re_link = Regex::new(
r#"document\.getElementById\('dlbutton'\)\.href = "(/d/.+/)" \+ \((\d+) % (\d+) \+ \d+ % (\d+)\) \+ "(.+)";"#,
)?;
let cap_link = match re_link.captures(&body) {
Some(cap) => cap,
None => return Err(Error::new(ErrorKind::Other, "Link not found").into())
None => return Err(Error::new(ErrorKind::Other, "Link not found").into()),
};
let url_start = &cap_link[1];
let url_end = &cap_link[5];
let n2: i32 = i32::from_str_radix(&cap_link[2], 10)?;