ffdl/src/main.rs

use std::path::Path;
use std::process::exit;
use clap::{ App, Arg, ArgGroup, crate_version };
use tokio::sync::mpsc;
use futures::future::join_all;
use std::io::BufRead;
use std::time::SystemTime;

use dlreport::{ DlReport, DlStatus, DlReporter };
use errors::ResBE;

mod zippy;
mod download;
mod errors;
mod dlreport;


#[tokio::main]
async fn main() -> ResBE<()> {

    let arguments = App::new("FDL - Fast/File Downloader")
        .version(crate_version!())
        .about("Download files fast")
        .arg(
            Arg::with_name("outdir")
                .short("o")
                .long("outdir")
                .value_name("OUTPUT DIR")
                .takes_value(true)
                .help("Set the output directory")
        )
        .arg(
            Arg::with_name("numdl")
                .short("n")
                .long("numdl")
                .value_name("NUMBER OF CONCURRENT DOWNLOADS")
                .takes_value(true)
                .help("Specify the number concurrent downloads")
        )
        .arg(
            Arg::with_name("boost")
                .short("b")
                .long("boost")
                .value_name("CONNECTIONS PER FILE")
                .takes_value(true)
                .help("Specify the number connections per single downloads. \
                        Files started with boost can't be continued. \
                        NOTE: This will likely cause IO bottlenecks on HDDs")
        )
        .arg(
            Arg::with_name("zippyshare")
                .short("z")
                .long("zippy")
                .takes_value(false)
                .help("The provided URLs are zippyshare URLs and need to be resolved")
        )
        .group(
            ArgGroup::with_name("action")
                .required(true)
        )
        .arg(
            Arg::with_name("listfile")
                .short("l")
                .long("listfile")
                .value_name("URL LIST")
                .takes_value(true)
                .group("action")
                .help("Download all files form the specified url list")
        )
        .arg(
            Arg::with_name("download")
                .short("d")
                .long("download")
                .value_name("URL")
                .takes_value(true)
                .group("action")
                .help("Download only the specified URL")
        )
        .arg(
            Arg::with_name("resolve")
                .short("r")
                .long("resolve")
                .value_name("URL")
                .takes_value(true)
                .group("action")
                .help("Resolve the zippyshare url to real download url")
        )
        .get_matches();
    

    

    let outdir = match arguments.value_of("outdir") {
        Some(it) => it,
        None => "./"
    };

    let numparal = match arguments.value_of("numdl") {
        Some(it) => it,
        None => "1"
    };

    let numparal: i32 = match numparal.parse() {
        Ok(it) => it,
        Err(_) => {
            eprintln!("Invalid value for numdl: {}", numparal);
            exit(1);
        }
    };
    

    let boost = match arguments.value_of("boost") {
        Some(it) => it,
        None => "1"
    };
    
    let boost: i32 = match boost.parse() {
        Ok(it) => it,
        Err(_) => {
            eprintln!("Invalid value for boost: {}", numparal);
            exit(1);
        }
    };

    let is_zippy = arguments.is_present("zippyshare");

    if arguments.is_present("listfile") {

        let listfile = arguments.value_of("listfile").unwrap();

        let ifile = std::fs::File::open(listfile)?;

        let urls: Vec<String> = std::io::BufReader::new(ifile)
            .lines()
            .map(|l| l.unwrap())
            .filter(|url| url.len() > 0 && !url.starts_with("#"))
            .collect();

        download_multiple(urls, outdir, numparal, boost, is_zippy).await?;
        
    } else if arguments.is_present("download") {

        let url = arguments.value_of("download").unwrap();

        let numparal = if boost != 1 {
            boost
        } else {
            numparal
        };

        download_multiple(vec![url.to_string()], outdir, 1, numparal, is_zippy).await?;

    } else if arguments.is_present("resolve") {

        let url = arguments.value_of("resolve").unwrap();

        match zippy::resolve_link(&url).await {
            Ok(resolved_url) => {
                println!("{}", resolved_url);
            },
            Err(e) => {
                println!("Zippyshare link could not be resolved");
                eprintln!("{}", e);
                exit(1);
            }

        }

    } else {
        println!("Something went very wrong...");
    }

    Ok(())
}

async fn download_multiple(urls: Vec<String>, outdir: &str, numparal: i32, boost: i32, is_zippy: bool) -> ResBE<()> {
    let outdir = Path::new(outdir);

    if !outdir.exists() {
        std::fs::create_dir_all(outdir)?;
    }

    let t_start = SystemTime::now();

    let mut joiners = Vec::new();

    let (tx, rx) = mpsc::unbounded_channel::<DlReport>();

    for offset in 0..numparal {

        let urls: Vec<String> = urls
            .iter()
            .enumerate()
            .filter(|(index, _)| (index) % numparal as usize == offset as usize)
            .map(|(_, v)| v.to_owned())
            .collect();
        
        let tx = tx.clone();
        let outdir = outdir.to_owned();
        let offset = offset;
        joiners.push(tokio::task::spawn(async move {

            for (i, url) in urls.iter().enumerate() {

                let tx = tx.clone();

                // Recalculated index in the main url vector, used as id
                let global_url_index = i as i32 * numparal + offset;

                let rep = DlReporter::new(global_url_index, tx);

                let url = if is_zippy {
                    match zippy::resolve_link(&url).await {
                        Ok(url) => url,
                        Err(_e) => {
                            rep.send(
                                DlStatus::MessageNow(format!("Zippyshare link could not be resolved: {}", url))
                            ).unwrap();

                            continue;
                        }
                    }
                } else {
                    url.to_string()
                };

                let file_name = download::url_to_filename(&url);
                let into_file = outdir.join(Path::new(&file_name))
                    .to_str().unwrap().to_string();
                let path_into_file = Path::new(&into_file);
            
                // If file with same name is present locally, check filesize
                if path_into_file.exists() {
                    let (filesize, _) = download::http_get_filesize_and_range_support(&url).await.unwrap();
                    let local_filesize = std::fs::metadata(path_into_file).unwrap().len();
            
                    if filesize == local_filesize {
                        rep.send(DlStatus::MessageNow(format!("Skipping file '{}': already present", &file_name))).unwrap();
                        continue;
                    } else {
                        rep.send(DlStatus::MessageNow(format!("Replacing file '{}': present but not completed", &file_name))).unwrap();
                    }
                }

                if boost == 1 {
                    if let Err(_e) = download::download_feedback(&url, &into_file, rep.clone()).await {
                        rep.send(DlStatus::DoneErr {
                            filename: into_file.to_string()
                        }).unwrap();
                    }
                } else {
                    if let Err(_e) = download::download_feedback_multi(&url, &into_file, rep.clone(), boost).await {
                        rep.send(DlStatus::DoneErr {
                            filename: into_file.to_string()
                        }).unwrap();
                    }
                };

            }
        }))

    }
    
    drop(tx);

    dlreport::watch_and_print_reports(rx).await?;

    join_all(joiners).await;

    println!("Total time: {}s", t_start.elapsed()?.as_secs());

    Ok(())
}