dnlmlr/ffdl
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Reduce the number of HTTP HEAD requests

Browse Source 
- Removed redundant http head requests that were used to query
  filesizes, even after the filesize was known already.
This commit is contained in:
Daniel M 2021-04-01 22:05:50 +02:00
parent 96fa5c4112
commit 39af87fcb4
3 changed files with 43 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/download.rs
View File

@ -74,20 +74,22 @@ pub fn url_to_filename(url: &str) -> String {
file_name.split("?").next().unwrap().to_string()
}
pub async fn download_feedback(url: &str, into_file: &str, rep: DlReporter) -> ResBE<()> {
pub async fn download_feedback(url: &str, into_file: &str, rep: DlReporter, content_length: Option<u64>) -> ResBE<()> {
download_feedback_chunks(url, into_file, rep, None, false).await
download_feedback_chunks(url, into_file, rep, None, false, content_length).await
}
pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporter, from_to: Option<(u64, u64)>, seek_from: bool) -> ResBE<()> {
pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporter, from_to: Option<(u64, u64)>, seek_from: bool, content_length: Option<u64>) -> ResBE<()> {
let into_file = Path::new(into_file);
let (mut content_length, range_supported) = http_get_filesize_and_range_support(url).await?;
if from_to != None && !range_supported{
return Err(DlError::Other("Server doesn't support range header".to_string()).into());
let mut content_length = match content_length {
Some(it) => it,
None => {
let (content_length, _) = http_get_filesize_and_range_support(url).await?;
content_length
}
};
// Send the HTTP request to download the given link
let mut req = reqwest::Client::new()
@ -223,9 +225,15 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte
// This will spin up multiple tasks that and manage the status updates for them.
// The combined status will be reported back to the caller
pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter, conn_count: i32) -> ResBE<()> {
pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter, conn_count: i32, content_length: Option<u64>) -> ResBE<()> {
let content_length = match content_length {
Some(it) => it,
None => {
let (content_length, _) = http_get_filesize_and_range_support(url).await?;
content_length
}
};
// Create zeroed file with 1 byte too much. This will be truncated on download
// completion and can indicate that the file is not suitable for continuation
@ -260,7 +268,9 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter
from_to.1 += rest;
}
download_feedback_chunks(&url, &into_file, rep, Some(from_to), true).await.map_err(|e| e.to_string())
let specific_content_length = from_to.1 - from_to.0 + 1;
download_feedback_chunks(&url, &into_file, rep, Some(from_to), true, Some(specific_content_length)).await.map_err(|e| e.to_string())
}))
}

1
src/errors.rs
View File

@ -4,6 +4,7 @@ use std::fmt::{ self, Display, Formatter };
/// Result Boxed Error
pub type ResBE<T> = Result<T, Box<dyn Error>>;
#[allow(unused)]
#[derive(Clone, Debug)]
pub enum DlError {
BadHttpStatus,

21
src/main.rs
View File

@ -240,10 +240,8 @@ async fn download_multiple(urls: Vec<String>, outdir: &str, file_count: i32, con
.to_str().unwrap().to_string();
let path_into_file = Path::new(&into_file);
// If file with same name is present locally, check filesize
if path_into_file.exists() {
let filesize = match download::http_get_filesize_and_range_support(&url).await {
Ok((filesize, _)) => filesize,
let (filesize, range_supported) = match download::http_get_filesize_and_range_support(&url).await {
Ok((filesize, range_supported)) => (filesize, range_supported),
Err(_e) => {
rep.send(
DlStatus::Message(format!("Error while querying metadata: {}", url))
@ -252,6 +250,9 @@ async fn download_multiple(urls: Vec<String>, outdir: &str, file_count: i32, con
}
};
// If file with same name is present locally, check filesize
if path_into_file.exists() {
let local_filesize = std::fs::metadata(path_into_file).unwrap().len();
if filesize == local_filesize {
@ -263,13 +264,21 @@ async fn download_multiple(urls: Vec<String>, outdir: &str, file_count: i32, con
}
if conn_count == 1 {
if let Err(_e) = download::download_feedback(&url, &into_file, rep.clone()).await {
if let Err(_e) = download::download_feedback(&url, &into_file, rep.clone(), Some(filesize)).await {
rep.send(DlStatus::DoneErr {
filename: into_file.to_string()
});
}
} else {
if let Err(_e) = download::download_feedback_multi(&url, &into_file, rep.clone(), conn_count).await {
if !range_supported {
rep.send(
DlStatus::Message(format!("Error Server does not support range header: {}", url))
);
continue;
}
if let Err(_e) = download::download_feedback_multi(&url, &into_file, rep.clone(), conn_count, Some(filesize)).await {
rep.send(DlStatus::DoneErr {
filename: into_file.to_string()
});