commit a8474aab1ed672798dbaa0a26a48c41674829c9c Author: Daniel M Date: Wed Mar 24 18:46:24 2021 +0100 Initial version diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..670c021 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/target +/tmp +/list.lst diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..cc9a4dd --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1249 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bumpalo" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "cc" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] +name = "crossterm" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c36c10130df424b2f3552fcc2ddcd9b28a27b1e54b358b45874f88d1ca6888c" +dependencies = [ + "bitflags", + "crossterm_winapi", + "lazy_static", + "libc", + "mio", + "parking_lot", + "signal-hook", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da8964ace4d3e4a044fd027919b2237000b24315a37c916f61809f1ff2140b9" +dependencies = [ + "winapi", +] + +[[package]] +name = "encoding_rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "fdl" +version = "0.1.0" +dependencies = [ + "bytes", + "clap", + "crossterm", + "futures", + "percent-encoding", + "regex", + "reqwest", + "tokio", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" + +[[package]] +name = "futures-executor" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" + +[[package]] +name = "futures-macro" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" + +[[package]] +name = "futures-task" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" + +[[package]] +name = "futures-util" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "h2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "hyper" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "js-sys" +version = "0.3.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc9f84f9b115ce7843d60706df1422a916680bfdfcbdb0447c5614ff9d7e4d78" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" + +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mio" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +dependencies = [ + "socket2", + "winapi", +] + +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10acf907b94fc1b1a152d08ef97e7759650268cf986bf127f387e602b02c7e5a" + +[[package]] +name = "openssl" +version = "0.10.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf12057f289428dbf5c591c74bf10392e4a8003f993405a902f20117019022d4" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfd318104249865096c8da1dfabf09ddbb6d0330ea176812a62ec75e40c4166" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee48cdde5ed250b0d3252818f646e174ab414036edb884dde62d80a3ac6082d" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" + +[[package]] +name = "serde_json" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43535db9747a4ba938c0ce0a98cc631a46ebf943c9e1d604e091df6007620bf6" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "signal-hook" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e31d442c16f047a671b5a71e2161d6e68814012b7f5379d269ebd915fac2729" +dependencies = [ + "libc", + "mio", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if", + "libc", + "winapi", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tinyvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "url" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasm-bindgen" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7" +dependencies = [ + "cfg-if", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b7d8b6942b8bb3a9b0e73fc79b98095a27de6fa247615e59d096754a3bc2aa8" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e67a5806118af01f0d9045915676b22aaebecf4178ae7021bc171dab0b897ab" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1" + +[[package]] +name = "web-sys" +version = "0.3.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec600b26223b2948cedfde2a0aa6756dcf1fef616f43d7b3097aaf53a6c4d92b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..43c9922 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "fdl" +version = "0.1.0" +authors = ["daniel m "] +edition = "2018" + +[dependencies] +tokio = { version = "1.2.0", features = [ "full" ] } +reqwest = { version = "0.11.2", features = [ "stream" ] } +futures = "0.3.12" +percent-encoding = "2.1.0" +regex = "1.4.3" +bytes = "1.0.1" +crossterm = "0.19.0" +clap = "2.33.3" +#futures-util = "0.3.13" diff --git a/src/dlreport.rs b/src/dlreport.rs new file mode 100644 index 0000000..fe2ba56 --- /dev/null +++ b/src/dlreport.rs @@ -0,0 +1,48 @@ +use tokio::sync::mpsc; + +use crate::errors::*; + + +#[derive(Clone, Debug)] +pub enum DlStatus { + Init { + bytes_total: u64, + filename: String + }, + Update { + speed_mbps: f64, + bytes_curr: u64 + }, + Done { + duration_ms: u64 + } +} + +#[derive(Clone, Debug)] +pub struct DlReport { + pub id: i32, + pub status: DlStatus +} + +pub struct DlReporter { + id: i32, + transmitter: mpsc::UnboundedSender +} + +impl DlReporter { + pub fn new(id: i32, transmitter: mpsc::UnboundedSender) -> DlReporter { + DlReporter { + id: id, + transmitter: transmitter + } + } + + pub fn send(& self, status: DlStatus) -> ResBE<()> { + self.transmitter.send( + DlReport { + id: self.id, + status: status + } + ).map_err(|e| e.into()) + } +} diff --git a/src/download.rs b/src/download.rs new file mode 100644 index 0000000..d9766e9 --- /dev/null +++ b/src/download.rs @@ -0,0 +1,277 @@ +use std::path::Path; +use tokio::io::AsyncWriteExt; +use std::time::SystemTime; +use percent_encoding::percent_decode_str; + +use crate::errors::*; +use crate::dlreport::*; + + +struct RollingAverage { + index: usize, + data: Vec +} + +impl RollingAverage { + + fn new(size: usize) -> Self { + RollingAverage { + index: 0, + data: Vec::with_capacity(size) + } + } + + fn value(&self) -> f64 { + if self.data.len() == 0 { + 0.0 + } else { + let sum: f64 = self.data.iter().sum(); + sum / self.data.len() as f64 + } + } + + fn add(&mut self, val: f64) { + if self.data.capacity() == self.data.len() { + + self.data[self.index] = val; + + self.index += 1; + if self.index >= self.data.capacity() { + self.index = 0; + } + + } else { + self.data.push(val); + } + } + +} + +/// Get the filename at the end of the given URL. This will decode the URL Encoding. +pub fn url_to_filename(url: &str) -> String { + let url_dec = percent_decode_str(&url).decode_utf8_lossy().to_owned().to_string(); + let file_name = std::path::Path::new(&url_dec).file_name().unwrap().to_str().unwrap(); + file_name.to_string() +} + +#[allow(unused)] +pub async fn download(url: &str, into_file: &str) -> ResBE<()> { + let into_file = Path::new(into_file); + + let mut resp = reqwest::Client::new() + .get(url) + .send().await?; + + let mut ofile = tokio::fs::OpenOptions::new() + // Open in write mode + .write(true) + // Delete and overwrite the file + .truncate(true) + // Create the file if not existant + .create(true) + .open(into_file).await?; + + // Read data from server as long as new data is available + while let Some(chunk) = resp.chunk().await? { + // Write the received data into the file + ofile.write_all(&chunk).await?; + } + + // Ensure that IO is completed + ofile.flush().await?; + + Ok(()) +} + +pub async fn download_feedback(url: &str, into_file: &str, rep: DlReporter) -> ResBE<()> { + let into_file = Path::new(into_file); + + // Send the HTTP request to download the given link + let mut resp = reqwest::Client::new() + .get(url) + .send().await?; + + // Error out if the server response is not success (something went wrong) + if !resp.status().is_success() { + return Err(DlError::BadHttpStatus.into()); + } + + // Get the content length for status update. If not present, error out cause + // without progress everything sucks anyways + let content_length = match resp.headers().get(reqwest::header::CONTENT_LENGTH) { + Some(cl) => cl.to_str()?.parse::()?, + None => return Err(DlError::ContentLengthUnknown.into()) + }; + + // Open the local output file + let mut ofile = tokio::fs::OpenOptions::new() + // Open in write mode + .write(true) + // Delete and overwrite the file + .truncate(true) + // Create the file if not existant + .create(true) + .open(into_file).await?; + + let filename = into_file.file_name().unwrap().to_str().unwrap(); + + // Report the download start + rep.send( + DlStatus::Init { + bytes_total: content_length, + filename: filename.to_string() + } + )?; + + + let mut curr_progress = 0; + let mut speed_mbps = 0.0; + + let t_start = SystemTime::now(); + + let mut t_last_speed = SystemTime::now(); + let mut last_bytecount = 0; + + let mut average_speed = RollingAverage::new(5); + + // Read data from server as long as new data is available + while let Some(chunk) = resp.chunk().await? { + + // Write the received data into the file + ofile.write_all(&chunk).await?; + + let datalen = chunk.len() as u64; + + // Update progress + curr_progress += datalen; + + // Update the number of bytes downloaded since the last report + last_bytecount += datalen; + + // Update the reported download speed after every 10MB + if last_bytecount > 10_000_000 { + let t_elapsed = t_last_speed.elapsed()?.as_millis(); + + // Update rolling average + average_speed.add( + (last_bytecount as f64) / (1000.0 * t_elapsed as f64) + ); + + speed_mbps = average_speed.value(); + + // Reset the time and bytecount + last_bytecount = 0; + t_last_speed = SystemTime::now(); + } + + // Send status update report + rep.send( + DlStatus::Update { + speed_mbps: speed_mbps, + bytes_curr: curr_progress + } + )?; + } + + // Ensure that IO is completed + ofile.flush().await?; + + let duration_ms = t_start.elapsed()?.as_millis() as u64; + + // Send report that the download is finished + rep.send( + DlStatus::Done { + duration_ms: duration_ms + } + )?; + + Ok(()) +} + +pub async fn http_get_filesize_and_range_support(url: &str) -> ResBE<(u64, bool)> { + let resp = reqwest::Client::new() + .head(url) + .send().await?; + + if let Some(filesize) = resp.headers().get(reqwest::header::CONTENT_LENGTH) { + if let Ok(val_str) = filesize.to_str() { + if let Ok(val) = val_str.parse::() { + + let mut range_supported = false; + + if let Some(range) = resp.headers().get(reqwest::header::ACCEPT_RANGES) { + if let Ok(range) = range.to_str() { + if range == "bytes" { + range_supported = true; + } + } + } + + return Ok((val, range_supported)); + } + } + } + + Err(DlError::ContentLengthUnknown.into()) +} + + + + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn rolling_average() { + let mut ra = RollingAverage::new(3); + + assert_eq!(0, ra.data.len()); + assert_eq!(3, ra.data.capacity()); + + assert_eq!(0.0, ra.value()); + + // 10 / 1 = 10 + ra.add(10.0); + assert_eq!(1, ra.data.len()); + assert_eq!(10.0, ra.value()); + + // (10 + 20) / 2 = 15 + ra.add(20.0); + assert_eq!(2, ra.data.len()); + assert_eq!(15.0, ra.value()); + + // (10 + 20 + 30) / 3 = 20 + ra.add(30.0); + assert_eq!(3, ra.data.len()); + assert_eq!(20.0, ra.value()); + + assert_eq!(10.0, ra.data[0]); + assert_eq!(20.0, ra.data[1]); + assert_eq!(30.0, ra.data[2]); + + // This should replace the oldest value (index 1) + ra.add(40.0); + + assert_eq!(3, ra.data.len()); + assert_eq!(3, ra.data.capacity()); + + // (40 + 20 + 30) / 3 = 30 + assert_eq!(30.0, ra.value()); + + assert_eq!(40.0, ra.data[0]); + assert_eq!(20.0, ra.data[1]); + assert_eq!(30.0, ra.data[2]); + + + ra.add(50.0); + ra.add(60.0); + ra.add(70.0); + + assert_eq!(70.0, ra.data[0]); + assert_eq!(50.0, ra.data[1]); + assert_eq!(60.0, ra.data[2]); + + } +} diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 0000000..ae06f1b --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,26 @@ +use std::error::Error; +use std::fmt::{ self, Display, Formatter }; + +/// Result Boxed Error +pub type ResBE = Result>; + +#[derive(Clone, Debug)] +pub enum DlError { + BadHttpStatus, + ContentLengthUnknown, + Other +} + +impl Error for DlError {} + +impl Display for DlError { + + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + DlError::BadHttpStatus => write!(f, "Bad http response status"), + DlError::ContentLengthUnknown => write!(f, "Content-Length is unknown"), + DlError::Other => write!(f, "Unknown download error") + } + } + +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..33c2567 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,404 @@ +use std::path::Path; +use std::process::exit; +use clap::{ App, Arg, ArgGroup, crate_version }; +use tokio::sync::mpsc; +use futures::future::join_all; +use std::time::SystemTime; +use std::io::BufRead; +use std::collections::HashMap; + +use dlreport::{ DlReport, DlStatus, DlReporter }; +use errors::ResBE; + +mod zippy; +mod download; +mod errors; +mod dlreport; + + +#[tokio::main] +async fn main() -> ResBE<()> { + + let arguments = App::new("FDL - Fast/File Downloader") + .version(crate_version!()) + .about("Download files fast") + .arg( + Arg::with_name("outdir") + .short("o") + .long("outdir") + .value_name("OUTPUT DIR") + .takes_value(true) + .help("Set the output directory") + ) + .arg( + Arg::with_name("numdl") + .short("n") + .long("numdl") + .value_name("NUMBER OF CONCURRENT DOWNLOADS") + .takes_value(true) + .help("Specify the number concurrent downloads") + ) + .arg( + Arg::with_name("zippyshare") + .short("z") + .long("zippy") + .takes_value(false) + .help("The provided URLs are zippyshare URLs and need to be resolved") + ) + .group( + ArgGroup::with_name("action") + .required(true) + ) + .arg( + Arg::with_name("listfile") + .short("l") + .long("listfile") + .value_name("URL LIST") + .takes_value(true) + .group("action") + .help("Download all files form the specified url list") + ) + .arg( + Arg::with_name("download") + .short("d") + .long("download") + .value_name("URL") + .takes_value(true) + .group("action") + .help("Download only the specified URL") + ) + .arg( + Arg::with_name("resolve") + .short("r") + .long("resolve") + .value_name("URL") + .takes_value(true) + .group("action") + .help("Resolve the zippyshare url to real download url") + ) + .get_matches(); + + + + + let outdir = match arguments.value_of("outdir") { + Some(it) => it, + None => "./" + }; + + let numparal = match arguments.value_of("numdl") { + Some(it) => it, + None => "1" + }; + + let numparal: i32 = match numparal.parse() { + Ok(it) => it, + Err(_) => { + eprintln!("Invalid value for numdl: {}", numparal); + exit(1); + } + }; + + let is_zippy = arguments.is_present("zippyshare"); + + if arguments.is_present("listfile") { + + let listfile = arguments.value_of("listfile").unwrap(); + + let ifile = std::fs::File::open(listfile)?; + + let mut urls: Vec = std::io::BufReader::new(ifile) + .lines() + .map(|l| l.unwrap()) + .filter(|url| url.len() > 0 && !url.starts_with("#")) + .collect(); + + if is_zippy { + let mut zippy_urls = Vec::new(); + for url in urls { + zippy_urls.push( + match zippy::resolve_link(&url).await { + Ok(url) => url, + Err(e) => { + println!("Zippyshare link could not be resolved"); + eprintln!("{}", e); + exit(1); + } + } + ) + } + + urls = zippy_urls; + } + + download_multiple(urls, outdir, numparal).await?; + + } else if arguments.is_present("download") { + + let url = arguments.value_of("download").unwrap(); + let url = if is_zippy { + match zippy::resolve_link(&url).await { + Ok(url) => url, + Err(e) => { + println!("Zippyshare link could not be resolved"); + eprintln!("{}", e); + exit(1); + } + } + } else { + url.to_string() + }; + + download_one(&url, outdir).await?; + + } else if arguments.is_present("resolve") { + + let url = arguments.value_of("resolve").unwrap(); + + match zippy::resolve_link(&url).await { + Ok(resolved_url) => { + println!("{}", resolved_url); + }, + Err(e) => { + println!("Zippyshare link could not be resolved"); + eprintln!("{}", e); + exit(1); + } + + } + + } else { + println!("Something went very wrong..."); + } + + Ok(()) +} + + +async fn download_one(url: &str, outdir: &str) -> ResBE<()> { + let outdir = Path::new(outdir); + + if !outdir.exists() { + std::fs::create_dir_all(outdir)?; + } + + + let file_name = download::url_to_filename(url); + let into_file = outdir.join(Path::new(&file_name)); + let into_file = into_file.to_str().unwrap().to_string(); + let path_into_file = Path::new(&into_file); + + // If file with same name is present locally, check filesize + if path_into_file.exists() { + let (filesize, _) = download::http_get_filesize_and_range_support(&url).await?; + let local_filesize = std::fs::metadata(path_into_file)?.len(); + + if filesize == local_filesize { + println!("Skipping file '{}': already present", &file_name); + return Ok(()); + } else { + println!("Replacing file '{}': present but not completed", &file_name); + } + } + + // Create com channel to get feedback on download progress + let (tx, mut rx) = mpsc::unbounded_channel::(); + + // Start download nonblocking + let url = url.to_string(); + let jh_download = tokio::spawn(async move { + // Create reporter with id 0 since there is only one anyways + let rep = DlReporter::new(0, tx); + + if let Err(e) = download::download_feedback(&url, &into_file, rep).await { + eprintln!("Error while downloading"); + eprintln!("{}", e); + } + }); + + + let mut t_last = SystemTime::UNIX_EPOCH; + + let mut filesize = 0; + + // Handle download status updates until all transmitters are closed + // this happens when the download is completed + while let Some(update) = rx.recv().await { + match update.status { + + DlStatus::Init { + bytes_total, + filename + } => { + + println!("Starting download for file '{}'", &filename); + filesize = bytes_total; + + }, + DlStatus::Update { + speed_mbps, + bytes_curr + } => { + + // Print update every second, otherwise ignore the updates + if t_last.elapsed()?.as_millis() > 1000 { + let percent_complete = bytes_curr as f64 / filesize as f64 * 100.0; + println!("Status: {:6.2} mb/s {:5.2}% completed", speed_mbps, percent_complete); + + t_last = SystemTime::now(); + } + + }, + DlStatus::Done { + duration_ms + } => { + + println!("Status: 100% completed"); + println!("Download took {} seconds", (duration_ms / 1000)); + + } + + } + } + + // Await the download just to make sure + jh_download.await?; + + Ok(()) +} + +async fn download_multiple(urls: Vec, outdir: &str, numparal: i32) -> ResBE<()> { + let outdir = Path::new(outdir); + + if !outdir.exists() { + std::fs::create_dir_all(outdir)?; + } + + let mut joiners = Vec::new(); + + let (tx, mut rx) = mpsc::unbounded_channel::(); + + for offset in 0..numparal { + + let urls: Vec = urls + .iter() + .enumerate() + .filter(|(index, _)| (index) % numparal as usize == offset as usize) + .map(|(_, v)| v.to_owned()) + .collect(); + + let tx = tx.clone(); + let outdir = outdir.to_owned(); + let offset = offset; + joiners.push(tokio::task::spawn(async move { + + for (i, url) in urls.iter().enumerate() { + + // Recalculated index in the main url vector, used as id + let global_url_index = i as i32 * numparal + offset; + + let file_name = download::url_to_filename(&url); + let into_file = outdir.join(Path::new(&file_name)); + let into_file = into_file.to_str().unwrap().to_string(); + let path_into_file = Path::new(&into_file); + + // If file with same name is present locally, check filesize + if path_into_file.exists() { + let (filesize, _) = download::http_get_filesize_and_range_support(&url).await.unwrap(); + let local_filesize = std::fs::metadata(path_into_file).unwrap().len(); + + if filesize == local_filesize { + println!("Skipping file '{}': already present", &file_name); + continue; + } else { + println!("Replacing file '{}': present but not completed", &file_name); + } + } + + let rep = DlReporter::new(global_url_index, tx.clone()); + + if let Err(e) = download::download_feedback(&url, &into_file, rep).await { + eprintln!("Error while downloading '{}'", file_name); + eprintln!("{}", e); + } + + } + })) + + } + + drop(tx); + + // filename, total size bytes, current size bytes, download speed mbps + let mut statuses: HashMap = HashMap::new(); + let mut t_last = SystemTime::now(); + + while let Some(update) = rx.recv().await { + match update.status { + + DlStatus::Init { + bytes_total, + filename + } => { + + println!("Starting download for file '{}'", &filename); + statuses.insert(update.id, (filename, bytes_total, 0, 0.0)); + + }, + DlStatus::Update { + speed_mbps, + bytes_curr + } => { + + // Scope the reference to prevent borrowing conflict later + { + let s = &mut statuses.get_mut(&update.id).unwrap(); + s.2 = bytes_curr; + s.3 = speed_mbps; + } + + if t_last.elapsed().unwrap().as_millis() > 2000 { + + let mut dl_speed_sum = 0.0; + + for (_k, v) in &statuses { + let filename = &v.0; + let filesize = v.1; + let bytes_curr = v.2; + let speed_mbps = v.3; + + let percent_complete = bytes_curr as f64 / filesize as f64 * 100.0; + println!("Status: {:6.2} mb/s {:5.2}% completed '{}'", speed_mbps, percent_complete, filename); + + dl_speed_sum += speed_mbps; + } + + println!("Accumulated download speed: {:6.2} mb/s\n", dl_speed_sum); + + t_last = SystemTime::now(); + + } + + }, + DlStatus::Done { + duration_ms + } => { + + println!( + "Status: 100% completed '{}'\nDownload took {} seconds", + &statuses.get(&update.id).unwrap().0, + (duration_ms / 1000) + ); + + statuses.remove(&update.id); + + } + + } + } + + join_all(joiners).await; + + + Ok(()) +} \ No newline at end of file diff --git a/src/zippy.rs b/src/zippy.rs new file mode 100644 index 0000000..2af40d7 --- /dev/null +++ b/src/zippy.rs @@ -0,0 +1,91 @@ +use regex::Regex; +use std::io::{ Error, ErrorKind }; + +use crate::errors::ResBE; + +#[allow(dead_code)] +pub async fn resolve_link_old(url: &str) -> ResBE { + + // Regex to check if the provided url is a zippyshare download url + let re = Regex::new(r"(https://www\d*\.zippyshare\.com)")?; + if !re.is_match(&url) { + return Err(Error::new(ErrorKind::Other, "URL is not a zippyshare url").into()); + } + + // Extract the hostname (with https:// prefix) for later + let base_host = &re.captures(&url).unwrap()[0]; + + // Download the html body for the download page + let body = reqwest::get(url).await? + .text().await?; + + // Regex to match the javascript part of the html that generates the real download link + let re = Regex::new(r#""(/d/\w+/)" \+ \((\d+) % (\d+) \+ (\d+) % (\d+)\) \+ "(/.+\.rar)";"#)?; + + if let Some(cap) = re.captures(&body) { + + // Extract the magic numbers used to generate the download link + let n1: i32 = i32::from_str_radix(&cap[2], 10)?; + let n2: i32 = i32::from_str_radix(&cap[3], 10)?; + let n3: i32 = i32::from_str_radix(&cap[4], 10)?; + let n4: i32 = i32::from_str_radix(&cap[5], 10)?; + + // Mix the numbers together + let mixed = n1 % n2 + n3 % n4; + + // Assemble the download link + let dl_url = format!("{}{}{}{}", base_host, &cap[1], mixed, &cap[6]); + + Ok(dl_url) + + } else { + Err(Error::new(ErrorKind::Other, "Link not found").into()) + } +} + +pub async fn resolve_link(url: &str) -> ResBE { + + // Regex to check if the provided url is a zippyshare download url + let re = Regex::new(r"(https://www\d*\.zippyshare\.com)")?; + if !re.is_match(&url) { + return Err(Error::new(ErrorKind::Other, "URL is not a zippyshare url").into()); + } + + // Extract the hostname (with https:// prefix) for later + let base_host = &re.captures(&url).unwrap()[0]; + + // Download the html body for the download page + let body = reqwest::get(url).await? + .text().await?; + + // Regex to match the javascript part of the html that generates the real download link + let re_a = Regex::new(r#"var a = (\d+);"#)?; + let re_b = Regex::new(r#"var b = (\d+);"#)?; + let re_concat = Regex::new(r#"document\.getElementById\('dlbutton'\)\.href = "(/d/.+/)"\+\(a \+ (\d+)%b\)\+"(/.+\.rar)";"#)?; + + let cap_a = match re_a.captures(&body) { + Some(cap) => cap, + None => return Err(Error::new(ErrorKind::Other, "Link not found").into()) + }; + + let cap_b = match re_b.captures(&body) { + Some(cap) => cap, + None => return Err(Error::new(ErrorKind::Other, "Link not found").into()) + }; + + let cap_concat = match re_concat.captures(&body) { + Some(cap) => cap, + None => return Err(Error::new(ErrorKind::Other, "Link not found").into()) + }; + + let a: i32 = i32::from_str_radix(&cap_a[1], 10)?; + let b: i32 = i32::from_str_radix(&cap_b[1], 10)?; + + let c: i32 = i32::from_str_radix(&cap_concat[2], 10)?; + + let mixed = (a/3) + (c%b); + + let dl_url = format!("{}{}{}{}", &base_host, &cap_concat[1], mixed, &cap_concat[3]); + + Ok(dl_url) +}