From 39af87fcb41a78e311eab42dc59a56ea6e2d1fd9 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Thu, 1 Apr 2021 22:05:50 +0200 Subject: [PATCH] Reduce the number of HTTP HEAD requests - Removed redundant http head requests that were used to query filesizes, even after the filesize was known already. --- src/download.rs | 34 ++++++++++++++++++++++------------ src/errors.rs | 1 + src/main.rs | 31 ++++++++++++++++++++----------- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/download.rs b/src/download.rs index 82226ea..9f0182f 100644 --- a/src/download.rs +++ b/src/download.rs @@ -74,20 +74,22 @@ pub fn url_to_filename(url: &str) -> String { file_name.split("?").next().unwrap().to_string() } -pub async fn download_feedback(url: &str, into_file: &str, rep: DlReporter) -> ResBE<()> { +pub async fn download_feedback(url: &str, into_file: &str, rep: DlReporter, content_length: Option) -> ResBE<()> { - download_feedback_chunks(url, into_file, rep, None, false).await + download_feedback_chunks(url, into_file, rep, None, false, content_length).await } -pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporter, from_to: Option<(u64, u64)>, seek_from: bool) -> ResBE<()> { +pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporter, from_to: Option<(u64, u64)>, seek_from: bool, content_length: Option) -> ResBE<()> { let into_file = Path::new(into_file); - - let (mut content_length, range_supported) = http_get_filesize_and_range_support(url).await?; - - if from_to != None && !range_supported{ - return Err(DlError::Other("Server doesn't support range header".to_string()).into()); - } + + let mut content_length = match content_length { + Some(it) => it, + None => { + let (content_length, _) = http_get_filesize_and_range_support(url).await?; + content_length + } + }; // Send the HTTP request to download the given link let mut req = reqwest::Client::new() @@ -223,9 +225,15 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte // This will spin up multiple tasks that and manage the status updates for them. // The combined status will be reported back to the caller -pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter, conn_count: i32) -> ResBE<()> { +pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter, conn_count: i32, content_length: Option) -> ResBE<()> { - let (content_length, _) = http_get_filesize_and_range_support(url).await?; + let content_length = match content_length { + Some(it) => it, + None => { + let (content_length, _) = http_get_filesize_and_range_support(url).await?; + content_length + } + }; // Create zeroed file with 1 byte too much. This will be truncated on download // completion and can indicate that the file is not suitable for continuation @@ -260,7 +268,9 @@ pub async fn download_feedback_multi(url: &str, into_file: &str, rep: DlReporter from_to.1 += rest; } - download_feedback_chunks(&url, &into_file, rep, Some(from_to), true).await.map_err(|e| e.to_string()) + let specific_content_length = from_to.1 - from_to.0 + 1; + + download_feedback_chunks(&url, &into_file, rep, Some(from_to), true, Some(specific_content_length)).await.map_err(|e| e.to_string()) })) } diff --git a/src/errors.rs b/src/errors.rs index 781e70a..6de17f1 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -4,6 +4,7 @@ use std::fmt::{ self, Display, Formatter }; /// Result Boxed Error pub type ResBE = Result>; +#[allow(unused)] #[derive(Clone, Debug)] pub enum DlError { BadHttpStatus, diff --git a/src/main.rs b/src/main.rs index ff8d194..53b8b9a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -239,18 +239,19 @@ async fn download_multiple(urls: Vec, outdir: &str, file_count: i32, con let into_file = outdir.join(Path::new(&file_name)) .to_str().unwrap().to_string(); let path_into_file = Path::new(&into_file); + + let (filesize, range_supported) = match download::http_get_filesize_and_range_support(&url).await { + Ok((filesize, range_supported)) => (filesize, range_supported), + Err(_e) => { + rep.send( + DlStatus::Message(format!("Error while querying metadata: {}", url)) + ); + continue; + } + }; // If file with same name is present locally, check filesize if path_into_file.exists() { - let filesize = match download::http_get_filesize_and_range_support(&url).await { - Ok((filesize, _)) => filesize, - Err(_e) => { - rep.send( - DlStatus::Message(format!("Error while querying metadata: {}", url)) - ); - continue; - } - }; let local_filesize = std::fs::metadata(path_into_file).unwrap().len(); @@ -263,13 +264,21 @@ async fn download_multiple(urls: Vec, outdir: &str, file_count: i32, con } if conn_count == 1 { - if let Err(_e) = download::download_feedback(&url, &into_file, rep.clone()).await { + if let Err(_e) = download::download_feedback(&url, &into_file, rep.clone(), Some(filesize)).await { rep.send(DlStatus::DoneErr { filename: into_file.to_string() }); } } else { - if let Err(_e) = download::download_feedback_multi(&url, &into_file, rep.clone(), conn_count).await { + + if !range_supported { + rep.send( + DlStatus::Message(format!("Error Server does not support range header: {}", url)) + ); + continue; + } + + if let Err(_e) = download::download_feedback_multi(&url, &into_file, rep.clone(), conn_count, Some(filesize)).await { rep.send(DlStatus::DoneErr { filename: into_file.to_string() });