diff --git a/src/download.rs b/src/download.rs index ff85096..7d6d6a9 100644 --- a/src/download.rs +++ b/src/download.rs @@ -57,36 +57,6 @@ pub fn url_to_filename(url: &str) -> String { file_name.to_string() } -#[allow(unused)] -pub async fn download(url: &str, into_file: &str) -> ResBE<()> { - let into_file = Path::new(into_file); - - let mut resp = reqwest::Client::new() - .get(url) - .send().await?; - - let mut ofile = tokio::fs::OpenOptions::new() - // Open in write mode - .write(true) - // Delete and overwrite the file - .truncate(true) - // Create the file if not existant - .create(true) - .open(into_file).await?; - - // Read data from server as long as new data is available - while let Some(chunk) = resp.chunk().await? { - // Write the received data into the file - ofile.write_all(&chunk).await?; - } - - // Ensure that IO is completed - ofile.flush().await?; - - Ok(()) -} - - pub async fn download_feedback(url: &str, into_file: &str, rep: DlReporter) -> ResBE<()> { download_feedback_chunks(url, into_file, rep, None, false).await @@ -161,14 +131,27 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte let mut average_speed = RollingAverage::new(10); + let mut buff: Vec = Vec::new(); + // Read data from server as long as new data is available while let Some(chunk) = resp.chunk().await? { - // Write the received data into the file - ofile.write_all(&chunk).await?; - let datalen = chunk.len() as u64; + buff.extend(chunk); + + // Buffer in memory first and only write to disk if the threshold is reached. + // This reduces the number of small disk writes and thereby reduces the + // io bottleneck that occurs on HDDs with many small writes in different + // files and offsets at the same time + if buff.len() >= 4_000_000 { + + // Write the received data into the file + ofile.write_all(&buff).await?; + + buff.clear(); + } + // Update progress curr_progress += datalen; @@ -199,6 +182,10 @@ pub async fn download_feedback_chunks(url: &str, into_file: &str, rep: DlReporte } )?; } + + if buff.len() > 0 { + ofile.write_all(&buff).await?; + } // Ensure that IO is completed //ofile.flush().await?;