Implement zippy resolver with js evaluation

- Add ducc crate (duktape bindings) as javascript engine - Extract the script tag from zippyshare page contents - Preprocess the script and execute the js to get the link - This removes the need for full regex based implementations for each antiscrape challenge
Update zippyshare resolver 2022-07-24 + bump
2022-09-13 15:37:20 +02:00 · 2022-08-16 21:42:08 +02:00 · 2022-07-24 15:39:00 +02:00 · 2022-07-17 23:34:19 +02:00 · 2022-07-17 23:33:21 +02:00 · 2022-07-17 23:24:11 +02:00
5 changed files with 277 additions and 133 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -13,9 +13,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.56"
+version = "1.0.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
+checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"

 [[package]]
 name = "atty"
@@ -64,6 +64,12 @@ version = "1.0.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"

+[[package]]
+name = "cesu8"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -85,16 +91,16 @@ dependencies = [

 [[package]]
 name = "clap"
-version = "3.1.6"
+version = "3.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8c93436c21e4698bacadf42917db28b23017027a4deccb35dbe47a7e7840123"
+checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db"
 dependencies = [
 "atty",
 "bitflags",
 "clap_derive",
+ "clap_lex",
 "indexmap",
 "lazy_static",
- "os_str_bytes",
 "strsim",
 "termcolor",
 "textwrap",
@@ -102,9 +108,9 @@ dependencies = [

 [[package]]
 name = "clap_derive"
-version = "3.1.4"
+version = "3.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da95d038ede1a964ce99f49cbe27a7fb538d1da595e4b4f70b8c8f338d17bf16"
+checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1"
 dependencies = [
 "heck",
 "proc-macro-error",
@@ -113,6 +119,15 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "clap_lex"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669"
+dependencies = [
+ "os_str_bytes",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.9.3"
@@ -131,14 +146,14 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"

 [[package]]
 name = "crossterm"
-version = "0.23.1"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1fd7173631a4e9e2ca8b32ae2fad58aab9843ea5aaf56642661937d87e28a3e"
+checksum = "ab9f7409c70a38a56216480fba371ee460207dd8926ccf5b4160591759559170"
 dependencies = [
 "bitflags",
 "crossterm_winapi",
 "libc",
- "mio 0.7.14",
+ "mio",
 "parking_lot",
 "signal-hook",
 "signal-hook-mio",
@@ -155,10 +170,29 @@ dependencies = [
 ]

 [[package]]
-name = "encoding_rs"
-version = "0.8.30"
+name = "ducc"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df"
+checksum = "41bc1f8a30712eb6a7454f85747f218d9dfb41d173bb223a8c4f18daff829207"
+dependencies = [
+ "cesu8",
+ "ducc-sys",
+]
+
+[[package]]
+name = "ducc-sys"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cdea834bf6a0fde522374db4404695c5f0465fc0ee814f2878d76eaabd4ffed"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
 dependencies = [
 "cfg-if",
 ]
@@ -174,12 +208,13 @@ dependencies = [

 [[package]]
 name = "ffdl"
-version = "0.1.2"
+version = "0.1.6"
 dependencies = [
 "anyhow",
 "chrono",
 "clap",
 "crossterm",
+ "ducc",
 "futures",
 "percent-encoding",
 "regex",
@@ -310,9 +345,9 @@ dependencies = [

 [[package]]
 name = "h2"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b"
+checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57"
 dependencies = [
 "bytes",
 "fnv",
@@ -323,7 +358,7 @@ dependencies = [
 "indexmap",
 "slab",
 "tokio",
- "tokio-util",
+ "tokio-util 0.7.1",
 "tracing",
 ]

@@ -372,9 +407,9 @@ dependencies = [

 [[package]]
 name = "httparse"
-version = "1.6.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4"
+checksum = "6330e8a36bd8c859f3fa6d9382911fbb7147ec39807f63b923933a247240b9ba"

 [[package]]
 name = "httpdate"
@@ -432,9 +467,9 @@ dependencies = [

 [[package]]
 name = "indexmap"
-version = "1.8.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
+checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee"
 dependencies = [
 "autocfg",
 "hashbrown",
@@ -451,9 +486,9 @@ dependencies = [

 [[package]]
 name = "ipnet"
-version = "2.4.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c"
+checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"

 [[package]]
 name = "itoa"
@@ -463,9 +498,9 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"

 [[package]]
 name = "js-sys"
-version = "0.3.56"
+version = "0.3.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04"
+checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397"
 dependencies = [
 "wasm-bindgen",
 ]
@@ -478,16 +513,17 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.121"
+version = "0.2.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f"
+checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"

 [[package]]
 name = "lock_api"
-version = "0.4.6"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b"
+checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
 dependencies = [
+ "autocfg",
 "scopeguard",
 ]

@@ -518,19 +554,6 @@ version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"

-[[package]]
-name = "mio"
-version = "0.7.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc"
-dependencies = [
- "libc",
- "log",
- "miow",
- "ntapi",
- "winapi",
-]
-
 [[package]]
 name = "mio"
 version = "0.8.2"
@@ -556,9 +579,9 @@ dependencies = [

 [[package]]
 name = "native-tls"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09bf6f32a3afefd0b587ee42ed19acd945c6d1f3b5424040f50b2f24ab16be77"
+checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9"
 dependencies = [
 "lazy_static",
 "libc",
@@ -654,9 +677,6 @@ name = "os_str_bytes"
 version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
-dependencies = [
- "memchr",
-]

 [[package]]
 name = "parking_lot"
@@ -670,9 +690,9 @@ dependencies = [

 [[package]]
 name = "parking_lot_core"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954"
+checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37"
 dependencies = [
 "cfg-if",
 "libc",
@@ -701,9 +721,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"

 [[package]]
 name = "pkg-config"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
+checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"

 [[package]]
 name = "proc-macro-error"
@@ -731,27 +751,27 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.36"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
+checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
 dependencies = [
 "unicode-xid",
 ]

 [[package]]
 name = "quote"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58"
+checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
 dependencies = [
 "proc-macro2",
 ]

 [[package]]
 name = "redox_syscall"
-version = "0.2.12"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0"
+checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
 dependencies = [
 "bitflags",
 ]
@@ -811,7 +831,7 @@ dependencies = [
 "serde_urlencoded",
 "tokio",
 "tokio-native-tls",
- "tokio-util",
+ "tokio-util 0.6.9",
 "url",
 "wasm-bindgen",
 "wasm-bindgen-futures",
@@ -910,7 +930,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
 dependencies = [
 "libc",
- "mio 0.7.14",
+ "mio",
 "signal-hook",
 ]

@@ -925,9 +945,9 @@ dependencies = [

 [[package]]
 name = "slab"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
+checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"

 [[package]]
 name = "smallvec"
@@ -953,9 +973,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"

 [[package]]
 name = "syn"
-version = "1.0.89"
+version = "1.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54"
+checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1024,9 +1044,9 @@ dependencies = [

 [[package]]
 name = "tinyvec"
-version = "1.5.1"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2"
+checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
 dependencies = [
 "tinyvec_macros",
 ]
@@ -1046,7 +1066,7 @@ dependencies = [
 "bytes",
 "libc",
 "memchr",
- "mio 0.8.2",
+ "mio",
 "num_cpus",
 "once_cell",
 "parking_lot",
@@ -1092,6 +1112,20 @@ dependencies = [
 "tokio",
 ]

+[[package]]
+name = "tokio-util"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "tower-service"
 version = "0.3.1"
@@ -1100,20 +1134,32 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"

 [[package]]
 name = "tracing"
-version = "0.1.32"
+version = "0.1.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"
+checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09"
 dependencies = [
 "cfg-if",
 "pin-project-lite",
+ "tracing-attributes",
 "tracing-core",
 ]

 [[package]]
-name = "tracing-core"
-version = "0.1.23"
+name = "tracing-attributes"
+version = "0.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c"
+checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f"
 dependencies = [
 "lazy_static",
 ]
@@ -1193,9 +1239,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"

 [[package]]
 name = "wasm-bindgen"
-version = "0.2.79"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
+checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad"
 dependencies = [
 "cfg-if",
 "wasm-bindgen-macro",
@@ -1203,9 +1249,9 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.79"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca"
+checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4"
 dependencies = [
 "bumpalo",
 "lazy_static",
@@ -1218,9 +1264,9 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.29"
+version = "0.4.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395"
+checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2"
 dependencies = [
 "cfg-if",
 "js-sys",
@@ -1230,9 +1276,9 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.79"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01"
+checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5"
 dependencies = [
 "quote",
 "wasm-bindgen-macro-support",
@@ -1240,9 +1286,9 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.79"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc"
+checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1253,15 +1299,15 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.79"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2"
+checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744"

 [[package]]
 name = "web-sys"
-version = "0.3.56"
+version = "0.3.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb"
+checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283"
 dependencies = [
 "js-sys",
 "wasm-bindgen",
@@ -1300,9 +1346,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

 [[package]]
 name = "windows-sys"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6"
+checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825"
 dependencies = [
 "windows_aarch64_msvc",
 "windows_i686_gnu",
@@ -1313,33 +1359,33 @@ dependencies = [

 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5"
+checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d"

 [[package]]
 name = "windows_i686_gnu"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615"
+checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed"

 [[package]]
 name = "windows_i686_msvc"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172"
+checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956"

 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc"
+checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4"

 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"
+checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9"

 [[package]]
 name = "winreg"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "ffdl"
-version = "0.1.2"
+version = "0.1.6"
 authors = ["daniel m <danielm@dnml.de>"]
 edition = "2021"
 description = "Download files fast"
@@ -11,8 +11,9 @@ reqwest = { version = "0.11.10", features = [ "stream" ] }
 futures = "0.3.21"
 percent-encoding = "2.1.0"
 regex = "1.5.5"
-crossterm = "0.23.1"
-clap = { version = "3.1.6", features = [ "derive" ] }
+clap = { version = "3.1.12", features = [ "derive" ] }
 chrono = "0.4.19"
 thiserror = "1.0.30"
-anyhow = "1.0.56"
+anyhow = "1.0.57"
+crossterm = "0.24.0"
+ducc = "0.1.5"
--- a/README.md
+++ b/README.md
@@ -0,0 +1,86 @@
+# Fast File Downloader
+A simple CLI application to download files over HTTP with the goal of maximum throuput by parallelizing downloads. Optimized for downloading large quantities of files from [zippyshare](https://zippyshare.com/) as fast as possible. 
+
+## Features
+* Automatically download files from a URL list
+* Automatically fetch actual download URLs for [zippyshare](https://zippyshare.com/) and [sendcm](https://send.cm/) links
+* Download multiple files at the same time
+* Use multiple connections to download non overlapping chunks of the same file in parallel
+  * This can be used to gain more download speed than a server would normally provide to a single connection
+  * Works really well with zippyshare
+  * This requires high random write performance for the local storage and will bottleneck on HDDs
+  * Only works if the host supports the HTTP Range header and doesn't otherwise block it
+
+## Performance
+On a 1000Mbit connection (actual speed ~800Mbit) I can download with about 90MB/s through a wireguard VPN from zippyshare when using 5 files in parallel with 10 connections each (`-n 5 -c 10`). 
+
+When downloading the same content manually through a browser I get about 5MB/s per file, dropping to 0.5-1MB/s after a few seconds. Also I of course have to start all the downloads by hand which is quite annoying.
+
+## Usage
+Simply create a textfile and enter the URLs one per line. Comments using `#` and empty lines are allowed.
+
+```
+$ cat list.txt
+# My encrypted documents backup
+https://www71.zippyshare.com/v/XXXXXXX/file.html
+https://www72.zippyshare.com/v/XXXXYYY/file.html
+https://www73.zippyshare.com/v/XXXXZZZ/file.html
+https://www74.zippyshare.com/v/Xagdgfh/file.html
+
+# My encrypted video backup
+https://www75.zippyshare.com/v/sdsgfds/file.html
+https://www76.zippyshare.com/v/dfsdxfd/file.html
+https://www75.zippyshare.com/v/dsgsdgf/file.html
+https://www76.zippyshare.com/v/drtdrtd/file.html
+https://www75.zippyshare.com/v/erdfghd/file.html
+https://www76.zippyshare.com/v/87654rd/file.html
+https://www75.zippyshare.com/v/dfghdfg/file.html
+https://www76.zippyshare.com/v/hkjghjk/file.html
+```
+
+And then download the files using the desired levels of concurrency (e.g. 5 files at once with 10 connections each):
+```
+$ ffdl -l list.txt -o ~/Downloads/ -c 10 -n 5
+```
+
+## CLI Arguments
+
+```
+$ ffdl --help
+
+FFDL - Fast File Downloader 0.1.2
+Download files fast
+
+USAGE:
+    ffdl.exe [OPTIONS]
+
+OPTIONS:
+    -c, --connections <CONNECTIONS PER FILE>
+            The number concurrent connections per file download. Increasing this number will
+            increase the download speed of individual files if supported by the server but setting
+            this number too high may cause the download to fail.
+            NOTE: This mode will write cause random writes and for that reason won't work on HDDs.
+            WARNING: Files started with multiple connections currently can't be continued. [default:
+            1]
+
+    -d, --download <URL>
+            Download only the one file from the specified url
+
+    -h, --help
+            Print help information
+
+    -l, --listfile <URL LISTFILE>
+            Download all files from the specified url list file
+
+    -n, --num-files <PARALLEL DOWNLOADS>
+            Specify the number of files from that should be downloaded in parallel. Increasing this
+            number will increase the total download speed but won't improve the download speed for
+            individual files [default: 1]
+
+    -o, --outdir <OUTPUT DIR>
+            Set the output directory in which the downloads will be stored. The directory will be
+            created if it doesn't exit yet [default: ./]
+
+    -V, --version
+            Print version information
+```
--- a/src/integrations/zippy.rs
+++ b/src/integrations/zippy.rs
@@ -1,61 +1,72 @@
 use std::io::{Error, ErrorKind};

 use anyhow::Result;
+use ducc::Ducc;
 use regex::Regex;

 pub fn is_zippyshare_url(url: &str) -> bool {
-    Regex::new(r"^https?://(?:www\d*\.)?zippyshare\.com/v/[0-9a-zA-Z]+/file\.html$")
+    Regex::new(r#"^https?://(?:www\d*\.)?zippyshare\.com/v/[0-9a-zA-Z]+/file\.html$"#)
        .unwrap()
        .is_match(url)
 }

-/*
-Updated: 07.03.2022
-Link generation code:
- `href = $1 + ($2 % $3 + $4 % $5) + $6`
- `$1` is always `/d/XXX` where XXX is dependent on the file
- `$2`, `$3`, `$4` and `$5` are dynamic and randomly generated on each reload
- `$2` is always the same as `$4`
- `$6` is dependent on the file
- The numbers in the calculation part ($2`, `$3`, `$4` and `$5`) are hard coded
-
-```
-document.getElementById('dlbutton').href = "/d/0Ky7p1C6/" + (186549 % 51245 + 186549 % 913) + "/some-file-name.part1.rar";
-```
-*/
 pub async fn resolve_link(url: &str) -> Result<String> {
    // Regex to check if the provided url is a zippyshare download url
-    let re = Regex::new(r"(https://www\d*\.zippyshare\.com)")?;
+    let re = Regex::new(r#"^(https?://(?:www\d*\.)?zippyshare\.com)/v/[0-9a-zA-Z]+/file\.html$"#)?;
    if !re.is_match(url) {
        return Err(Error::new(ErrorKind::Other, "URL is not a zippyshare url").into());
    }

    // Extract the hostname (with https:// prefix) for later
-    let base_host = &re.captures(url).unwrap()[0];
+    let host = &re.captures(url).unwrap()[1];

    // Download the html body for the download page
    let body = reqwest::get(url).await?.text().await?;

-    // Regex to match the javascript part of the html that generates the real download link
-    let re_link = Regex::new(
-        r#"document\.getElementById\('dlbutton'\)\.href = "(/d/.+/)" \+ \((\d+) % (\d+) \+ \d+ % (\d+)\) \+ "(.+)";"#,
-    )?;
+    let re_script =
+        Regex::new(r#"(?ms)<script.*?>(.*getElementById\('dlbutton'\).*?)</script>"#).unwrap();
+    let re_script_start = Regex::new(r#"(?ms)<script.*?>"#).unwrap();

-    let cap_link = match re_link.captures(&body) {
+    // Extract the script. This will end at the correct script end, but has stuff before the start
+    let cap_tmp = match re_script.captures(&body) {
        Some(cap) => cap,
        None => return Err(Error::new(ErrorKind::Other, "Link not found").into()),
    };
+    let temp = &cap_tmp[1];

-    let url_start = &cap_link[1];
-    let url_end = &cap_link[5];
-    let n2: i32 = cap_link[2].parse()?;
-    let n3: i32 = cap_link[3].parse()?;
-    let n4 = n2;
-    let n5: i32 = cap_link[4].parse()?;
+    // Find the correct script start
+    let pos_script_start = match re_script_start.find_iter(&temp).last() {
+        Some(cap) => cap,
+        None => return Err(Error::new(ErrorKind::Other, "Link not found").into()),
+    };
+    // Cut off the beginning to get only the script contents
+    let raw_script = &temp[pos_script_start.end()..];

-    let mixed = n2 % n3 + n4 % n5;
+    // Preprocess the script
+    let script = preprocess_js(raw_script);

-    let dl_url = format!("{}{}{}{}", &base_host, url_start, mixed, url_end);
+    // Calculate the link
+    let link = eval_js_link_calculation(&script)
+        .map_err(|_| Error::new(ErrorKind::Other, "Link not found: JS eval error"))?;

-    Ok(dl_url)
+    let url = format!("{}{}", host, link);
+    Ok(url)
+}
+
+fn preprocess_js(js_src: &str) -> String {
+    let mut processed_src = js_src
+        .replace("document.getElementById('dlbutton').href", "href")
+        .replace("document.getElementById('fimage')", "false")
+        // Fix for antiscrape 24.07.2022
+        .replace("document.getElementById('omg').getAttribute('class')", "2")
+        // Fix for antiscrape 16.08.2022
+        .replace("document.getElementById('dlbutton').omg", "omg");
+
+    processed_src.push_str(";href");
+    processed_src
+}
+
+fn eval_js_link_calculation(js_src: &str) -> ducc::Result<String> {
+    let ducc = Ducc::new();
+    ducc.exec(js_src, None, Default::default())
 }
--- a/src/main.rs
+++ b/src/main.rs
@@ -114,8 +114,8 @@ async fn download_job(urls: SyncQueue, reporter: UnboundedSender<DlReport>, cli_
        let mut urls = urls.lock().await;
        urls.pop_front().take()
    } {
-        println!("Downloading {}", dlreq.url);
        let reporter = DlReporter::new(dlreq.id as u32, reporter.clone());
+        report_msg!(reporter, "Downloading {}", dlreq.url);

        // Resolve the zippy url to the direct download url if necessary
        let url = match is_integrated_url(&dlreq.url) {
Author	SHA1	Message	Date
Daniel M	f9659246ca	Implement zippy resolver with js evaluation - Add ducc crate (duktape bindings) as javascript engine - Extract the script tag from zippyshare page contents - Preprocess the script and execute the js to get the link - This removes the need for full regex based implementations for each antiscrape challenge	2022-09-13 15:37:20 +02:00
Daniel M	bc2d312ce9	Update zippyshare resolver 2022-07-24 + bump - Bump version to 0.1.6	2022-08-16 21:42:08 +02:00
Daniel M	0f7e05a71d	Update zippyshare resolver 2022-07-24 + bump - Bump version to 0.1.5	2022-07-24 15:39:00 +02:00
Daniel M	2e0c12ee56	Bump version to 0.1.4	2022-07-17 23:34:19 +02:00
Daniel M	7606f90384	Update crossterm dependency - Crossterm 0.24 was released with the merged bugfix, so no need for the pinned git dependency anymore	2022-07-17 23:33:21 +02:00
Daniel M	33d772c9e3	Update zippyshare resolver 2022-07-17	2022-07-17 23:24:11 +02:00
Daniel M	e7dca54b2b	Add README	2022-06-16 01:40:11 +02:00
Daniel M	4fb19a4f5b	Use latest git version for crossterm to fix bug - Fix visual bugs on windows using git-bash that are caused by crossterm - The crossterm bug was fixed in upstream by PR-657 but is not yet released to crates.io - Bump version to 0.1.3	2022-06-15 17:51:07 +02:00
Daniel M	276ff194db	Fix broken terminal output - Replaced println by report message call	2022-04-26 00:21:29 +02:00