Fix sendcm, update dependencies, version 0.1.8

- Sendcmd CDN requires an extra referer header from itself, otherwise it
  404s the requests
- Update dependencies except clap
- Bump version to 0.1.8

Cargo.lock

@@ -12,10 +12,19 @@ dependencies = [
 ]
 
 [[package]]
-name = "anyhow"
+name = "android_system_properties"
-version = "1.0.56"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
 
 [[package]]
 name = "atty"
@@ -64,6 +73,12 @@ version = "1.0.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
 
+[[package]]
+name = "cesu8"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -72,29 +87,31 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.19"
+version = "0.4.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
+checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
 dependencies = [
- "libc",
+ "iana-time-zone",
+ "js-sys",
  "num-integer",
  "num-traits",
  "time",
+ "wasm-bindgen",
  "winapi",
 ]
 
 [[package]]
 name = "clap"
-version = "3.1.6"
+version = "3.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8c93436c21e4698bacadf42917db28b23017027a4deccb35dbe47a7e7840123"
+checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db"
 dependencies = [
  "atty",
  "bitflags",
  "clap_derive",
+ "clap_lex",
  "indexmap",
  "lazy_static",
- "os_str_bytes",
  "strsim",
  "termcolor",
  "textwrap",
@@ -102,9 +119,9 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "3.1.4"
+version = "3.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da95d038ede1a964ce99f49cbe27a7fb538d1da595e4b4f70b8c8f338d17bf16"
+checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1"
 dependencies = [
  "heck",
  "proc-macro-error",
@@ -113,6 +130,25 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "clap_lex"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669"
+dependencies = [
+ "os_str_bytes",
+]
+
+[[package]]
+name = "codespan-reporting"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
+dependencies = [
+ "termcolor",
+ "unicode-width",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.9.3"
@@ -131,14 +167,14 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
 
 [[package]]
 name = "crossterm"
-version = "0.23.1"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1fd7173631a4e9e2ca8b32ae2fad58aab9843ea5aaf56642661937d87e28a3e"
+checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
 dependencies = [
  "bitflags",
  "crossterm_winapi",
  "libc",
- "mio 0.7.14",
+ "mio",
  "parking_lot",
  "signal-hook",
  "signal-hook-mio",
@@ -155,10 +191,73 @@ dependencies = [
 ]
 
 [[package]]
-name = "encoding_rs"
+name = "cxx"
-version = "0.8.30"
+version = "1.0.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df"
+checksum = "7e599641dff337570f6aa9c304ecca92341d30bf72e1c50287869ed6a36615a6"
+dependencies = [
+ "cc",
+ "cxxbridge-flags",
+ "cxxbridge-macro",
+ "link-cplusplus",
+]
+
+[[package]]
+name = "cxx-build"
+version = "1.0.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60e2434bc22249c056e12d2e87db46380730da0f2648471edea3e8e11834a892"
+dependencies = [
+ "cc",
+ "codespan-reporting",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "scratch",
+ "syn",
+]
+
+[[package]]
+name = "cxxbridge-flags"
+version = "1.0.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3894ad0c6d517cb5a4ce8ec20b37cd0ea31b480fe582a104c5db67ae21270853"
+
+[[package]]
+name = "cxxbridge-macro"
+version = "1.0.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34fa7e395dc1c001083c7eed28c8f0f0b5a225610f3b6284675f444af6fab86b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "ducc"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41bc1f8a30712eb6a7454f85747f218d9dfb41d173bb223a8c4f18daff829207"
+dependencies = [
+ "cesu8",
+ "ducc-sys",
+]
+
+[[package]]
+name = "ducc-sys"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cdea834bf6a0fde522374db4404695c5f0465fc0ee814f2878d76eaabd4ffed"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
 dependencies = [
  "cfg-if",
 ]
@@ -174,12 +273,13 @@ dependencies = [
 
 [[package]]
 name = "ffdl"
-version = "0.1.2"
+version = "0.1.8"
 dependencies = [
  "anyhow",
  "chrono",
  "clap",
  "crossterm",
+ "ducc",
  "futures",
  "percent-encoding",
  "regex",
@@ -221,9 +321,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e"
+checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -236,9 +336,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010"
+checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -246,15 +346,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3"
+checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6"
+checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -263,15 +363,15 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
+checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512"
+checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -280,21 +380,21 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868"
+checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
 
 [[package]]
 name = "futures-task"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a"
+checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
 
 [[package]]
 name = "futures-util"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a"
+checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -310,9 +410,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b"
+checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57"
 dependencies = [
  "bytes",
  "fnv",
@@ -372,9 +472,9 @@ dependencies = [
 
 [[package]]
 name = "httparse"
-version = "1.6.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4"
+checksum = "6330e8a36bd8c859f3fa6d9382911fbb7147ec39807f63b923933a247240b9ba"
 
 [[package]]
 name = "httpdate"
@@ -419,6 +519,30 @@ dependencies = [
  "tokio-native-tls",
 ]
 
+[[package]]
+name = "iana-time-zone"
+version = "0.1.53"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "wasm-bindgen",
+ "winapi",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
+dependencies = [
+ "cxx",
+ "cxx-build",
+]
+
 [[package]]
 name = "idna"
 version = "0.2.3"
@@ -432,9 +556,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "1.8.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
+checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee"
 dependencies = [
  "autocfg",
  "hashbrown",
@@ -451,9 +575,9 @@ dependencies = [
 
 [[package]]
 name = "ipnet"
-version = "2.4.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c"
+checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
 
 [[package]]
 name = "itoa"
@@ -463,9 +587,9 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
 
 [[package]]
 name = "js-sys"
-version = "0.3.56"
+version = "0.3.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04"
+checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -478,16 +602,26 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.121"
+version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f"
+checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+
+[[package]]
+name = "link-cplusplus"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
+dependencies = [
+ "cc",
+]
 
 [[package]]
 name = "lock_api"
-version = "0.4.6"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b"
+checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
 dependencies = [
+ "autocfg",
  "scopeguard",
 ]
 
@@ -520,45 +654,21 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
 [[package]]
 name = "mio"
-version = "0.7.14"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc"
+checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
 dependencies = [
  "libc",
  "log",
- "miow",
- "ntapi",
- "winapi",
-]
-
-[[package]]
-name = "mio"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9"
-dependencies = [
- "libc",
- "log",
- "miow",
- "ntapi",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "winapi",
+ "windows-sys 0.42.0",
-]
-
-[[package]]
-name = "miow"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
-dependencies = [
- "winapi",
 ]
 
 [[package]]
 name = "native-tls"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09bf6f32a3afefd0b587ee42ed19acd945c6d1f3b5424040f50b2f24ab16be77"
+checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9"
 dependencies = [
  "lazy_static",
  "libc",
@@ -572,15 +682,6 @@ dependencies = [
  "tempfile",
 ]
 
-[[package]]
-name = "ntapi"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "num-integer"
 version = "0.1.44"
@@ -654,9 +755,6 @@ name = "os_str_bytes"
 version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
-dependencies = [
- "memchr",
-]
 
 [[package]]
 name = "parking_lot"
@@ -670,22 +768,22 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954"
+checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37"
 dependencies = [
  "cfg-if",
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-sys",
+ "windows-sys 0.34.0",
 ]
 
 [[package]]
 name = "percent-encoding"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
+checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
 
 [[package]]
 name = "pin-project-lite"
@@ -701,9 +799,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pkg-config"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
+checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
 
 [[package]]
 name = "proc-macro-error"
@@ -731,36 +829,36 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.36"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
+checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
 dependencies = [
  "unicode-xid",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58"
+checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
 dependencies = [
  "proc-macro2",
 ]
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.12"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0"
+checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
 dependencies = [
  "bitflags",
 ]
 
 [[package]]
 name = "regex"
-version = "1.5.5"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
+checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -769,9 +867,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.25"
+version = "0.6.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
+checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
 
 [[package]]
 name = "remove_dir_all"
@@ -784,9 +882,9 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.10"
+version = "0.11.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb"
+checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c"
 dependencies = [
  "base64",
  "bytes",
@@ -800,10 +898,10 @@ dependencies = [
  "hyper-tls",
  "ipnet",
  "js-sys",
- "lazy_static",
  "log",
  "mime",
  "native-tls",
+ "once_cell",
  "percent-encoding",
  "pin-project-lite",
  "serde",
@@ -812,6 +910,7 @@ dependencies = [
  "tokio",
  "tokio-native-tls",
  "tokio-util",
+ "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
@@ -841,6 +940,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "scratch"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2"
+
 [[package]]
 name = "security-framework"
 version = "2.6.1"
@@ -910,7 +1015,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
 dependencies = [
  "libc",
- "mio 0.7.14",
+ "mio",
  "signal-hook",
 ]
 
@@ -925,9 +1030,9 @@ dependencies = [
 
 [[package]]
 name = "slab"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
+checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"
 
 [[package]]
 name = "smallvec"
@@ -953,9 +1058,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
-version = "1.0.89"
+version = "1.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54"
+checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -993,18 +1098,18 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
 
 [[package]]
 name = "thiserror"
-version = "1.0.30"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
+checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.30"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
+checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1024,9 +1129,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.5.1"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2"
+checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -1039,22 +1144,22 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.17.0"
+version = "1.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"
+checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae"
 dependencies = [
+ "autocfg",
  "bytes",
  "libc",
  "memchr",
- "mio 0.8.2",
+ "mio",
  "num_cpus",
- "once_cell",
  "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "winapi",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -1080,16 +1185,16 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.6.9"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"
+checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764"
 dependencies = [
  "bytes",
  "futures-core",
  "futures-sink",
- "log",
  "pin-project-lite",
  "tokio",
+ "tracing",
 ]
 
 [[package]]
@@ -1100,20 +1205,32 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
 
 [[package]]
 name = "tracing"
-version = "0.1.32"
+version = "0.1.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"
+checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09"
 dependencies = [
  "cfg-if",
  "pin-project-lite",
+ "tracing-attributes",
  "tracing-core",
 ]
 
 [[package]]
-name = "tracing-core"
+name = "tracing-attributes"
-version = "0.1.23"
+version = "0.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c"
+checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f"
 dependencies = [
  "lazy_static",
 ]
@@ -1139,6 +1256,12 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unicode-width"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.2"
@@ -1193,9 +1316,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.79"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
+checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -1203,9 +1326,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.79"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca"
+checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4"
 dependencies = [
  "bumpalo",
  "lazy_static",
@@ -1218,9 +1341,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.29"
+version = "0.4.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395"
+checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -1230,9 +1353,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.79"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01"
+checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -1240,9 +1363,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.79"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc"
+checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1253,15 +1376,15 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.79"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2"
+checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744"
 
 [[package]]
 name = "web-sys"
-version = "0.3.56"
+version = "0.3.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb"
+checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -1300,46 +1423,103 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windows-sys"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6"
+checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825"
 dependencies = [
- "windows_aarch64_msvc",
+ "windows_aarch64_msvc 0.34.0",
- "windows_i686_gnu",
+ "windows_i686_gnu 0.34.0",
- "windows_i686_msvc",
+ "windows_i686_msvc 0.34.0",
- "windows_x86_64_gnu",
+ "windows_x86_64_gnu 0.34.0",
- "windows_x86_64_msvc",
+ "windows_x86_64_msvc 0.34.0",
 ]
 
 [[package]]
-name = "windows_aarch64_msvc"
+name = "windows-sys"
-version = "0.32.0"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5"
+checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc 0.42.1",
+ "windows_i686_gnu 0.42.1",
+ "windows_i686_msvc 0.42.1",
+ "windows_x86_64_gnu 0.42.1",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc 0.42.1",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615"
+checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172"
+checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc"
+checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.32.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"
+checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
 [[package]]
 name = "winreg"

Cargo.toml

@@ -1,18 +1,19 @@
 [package]
 name = "ffdl"
-version = "0.1.2"
+version = "0.1.8"
 authors = ["daniel m <danielm@dnml.de>"]
-edition = "2018"
+edition = "2021"
 description = "Download files fast"
 
 [dependencies]
-tokio = { version = "1.17.0", features = [ "full" ] }
+tokio = { version = "1.24.1", features = [ "full" ] }
-reqwest = { version = "0.11.10", features = [ "stream" ] }
+reqwest = { version = "0.11.13", features = [ "stream" ] }
-futures = "0.3.21"
+futures = "0.3.25"
-percent-encoding = "2.1.0"
+percent-encoding = "2.2.0"
-regex = "1.5.5"
+regex = "1.7.1"
-crossterm = "0.23.1"
+clap = { version = "3.1.12", features = [ "derive" ] }
-clap = { version = "3.1.6", features = [ "derive" ] }
+chrono = "0.4.23"
-chrono = "0.4.19"
+thiserror = "1.0.38"
-thiserror = "1.0.30"
+anyhow = "1.0.68"
-anyhow = "1.0.56"
+crossterm = "0.25.0"
+ducc = "0.1.5"

README.md

@@ -0,0 +1,86 @@
+# Fast File Downloader
+A simple CLI application to download files over HTTP with the goal of maximum throuput by parallelizing downloads. Optimized for downloading large quantities of files from [zippyshare](https://zippyshare.com/) as fast as possible. 
+
+## Features
+* Automatically download files from a URL list
+* Automatically fetch actual download URLs for [zippyshare](https://zippyshare.com/) and [sendcm](https://send.cm/) links
+* Download multiple files at the same time
+* Use multiple connections to download non overlapping chunks of the same file in parallel
+  * This can be used to gain more download speed than a server would normally provide to a single connection
+  * Works really well with zippyshare
+  * This requires high random write performance for the local storage and will bottleneck on HDDs
+  * Only works if the host supports the HTTP Range header and doesn't otherwise block it
+
+## Performance
+On a 1000Mbit connection (actual speed ~800Mbit) I can download with about 90MB/s through a wireguard VPN from zippyshare when using 5 files in parallel with 10 connections each (`-n 5 -c 10`). 
+
+When downloading the same content manually through a browser I get about 5MB/s per file, dropping to 0.5-1MB/s after a few seconds. Also I of course have to start all the downloads by hand which is quite annoying.
+
+## Usage
+Simply create a textfile and enter the URLs one per line. Comments using `#` and empty lines are allowed.
+
+```
+$ cat list.txt
+# My encrypted documents backup
+https://www71.zippyshare.com/v/XXXXXXX/file.html
+https://www72.zippyshare.com/v/XXXXYYY/file.html
+https://www73.zippyshare.com/v/XXXXZZZ/file.html
+https://www74.zippyshare.com/v/Xagdgfh/file.html
+
+# My encrypted video backup
+https://www75.zippyshare.com/v/sdsgfds/file.html
+https://www76.zippyshare.com/v/dfsdxfd/file.html
+https://www75.zippyshare.com/v/dsgsdgf/file.html
+https://www76.zippyshare.com/v/drtdrtd/file.html
+https://www75.zippyshare.com/v/erdfghd/file.html
+https://www76.zippyshare.com/v/87654rd/file.html
+https://www75.zippyshare.com/v/dfghdfg/file.html
+https://www76.zippyshare.com/v/hkjghjk/file.html
+```
+
+And then download the files using the desired levels of concurrency (e.g. 5 files at once with 10 connections each):
+```
+$ ffdl -l list.txt -o ~/Downloads/ -c 10 -n 5
+```
+
+## CLI Arguments
+
+```
+$ ffdl --help
+
+FFDL - Fast File Downloader 0.1.2
+Download files fast
+
+USAGE:
+    ffdl.exe [OPTIONS]
+
+OPTIONS:
+    -c, --connections <CONNECTIONS PER FILE>
+            The number concurrent connections per file download. Increasing this number will
+            increase the download speed of individual files if supported by the server but setting
+            this number too high may cause the download to fail.
+            NOTE: This mode will write cause random writes and for that reason won't work on HDDs.
+            WARNING: Files started with multiple connections currently can't be continued. [default:
+            1]
+
+    -d, --download <URL>
+            Download only the one file from the specified url
+
+    -h, --help
+            Print help information
+
+    -l, --listfile <URL LISTFILE>
+            Download all files from the specified url list file
+
+    -n, --num-files <PARALLEL DOWNLOADS>
+            Specify the number of files from that should be downloaded in parallel. Increasing this
+            number will increase the total download speed but won't improve the download speed for
+            individual files [default: 1]
+
+    -o, --outdir <OUTPUT DIR>
+            Set the output directory in which the downloads will be stored. The directory will be
+            created if it doesn't exit yet [default: ./]
+
+    -V, --version
+            Print version information
+```

src/download.rs

@@ -10,6 +10,7 @@ use tokio::io::{AsyncSeekExt, AsyncWriteExt};
 
 use crate::dlreport::{DlReportAccumulator, DlReporter};
 use crate::errors::DlError;
+use crate::integrations::HttpRequestPreprocessor;
 use crate::misc::RollingAverage;
 
 /// Get the filename at the end of the given URL. This will decode the URL Encoding.
@@ -45,6 +46,7 @@ pub async fn download_feedback_chunks(
 ) -> Result<()> {
     // Build the HTTP request to download the given link
     let mut req = reqwest::Client::new().get(url);
+    req = req.integration_preprocess(url);
 
     // Add range header if needed
     if let Some((from, to)) = from_to {
@@ -267,7 +269,9 @@ pub struct HttpFileInfo {
 }
 
 pub async fn http_file_info(url: &str) -> Result<HttpFileInfo> {
-    let resp = reqwest::Client::new().head(url).send().await?;
+    let mut req = reqwest::Client::new().head(url);
+    req = req.integration_preprocess(url);
+    let resp = req.send().await?;
 
     let filesize = resp
         .headers()

src/integrations.rs

@@ -0,0 +1,46 @@
+mod sendcm;
+mod zippy;
+
+use anyhow::Result;
+
+pub enum IntegratedService {
+    ZippyShare,
+    SendCm,
+}
+
+pub fn is_integrated_url(url: &str) -> Option<IntegratedService> {
+    if zippy::is_zippyshare_url(url) {
+        Some(IntegratedService::ZippyShare)
+    } else if sendcm::is_sendcm_url(url) {
+        Some(IntegratedService::SendCm)
+    } else {
+        None
+    }
+}
+
+pub async fn resolve_integrated_url(url: &str, service: IntegratedService) -> Result<String> {
+    match service {
+        IntegratedService::ZippyShare => zippy::resolve_link(url).await,
+        IntegratedService::SendCm => sendcm::resolve_link(url).await,
+    }
+}
+
+pub trait HttpRequestPreprocessor {
+    /// Add required headers or other modifications to an HTTP request depending on the target
+    /// website
+    fn integration_preprocess(self, url: &str) -> Self;
+}
+
+impl HttpRequestPreprocessor for reqwest::RequestBuilder {
+    fn integration_preprocess(self, url: &str) -> Self {
+        // The sendcm CDN (sendit.cloud) requires a referer from itself to allow access and not 404
+        if url.contains("sendit.cloud") {
+            match url.split("/d/").next() {
+                Some(base_url) => self.header(reqwest::header::REFERER, format!("{}/", base_url)),
+                None => self,
+            }
+        } else {
+            self
+        }
+    }
+}

src/integrations/sendcm.rs

@@ -0,0 +1,84 @@
+use std::io::{Error, ErrorKind};
+
+use anyhow::Result;
+use regex::Regex;
+
+pub fn is_sendcm_url(url: &str) -> bool {
+    Regex::new(r"^https?://send\.cm/(?:d/)?[0-9a-zA-Z]+$")
+        .unwrap()
+        .is_match(url)
+}
+
+/*
+Updated: 01.04.2022
+Link generation code:
+- A post request is sent to the server using the form described below
+- The id field is the value which is used to generate the link
+- If the id is not found, the link is not generated
+- The id is the same as the url suffix when NOT using a /d/ prefix url
+- The reponse to the post request is a 302 redirect to the generated link
+
+```
+<form name="F1" method="POST" action="https://send.cm">
+<input type="hidden" name="op" value="download2">
+<input type="hidden" name="id" value="xxxxxxxxxx">
+<input type="hidden" name="rand" value="">
+<input type="hidden" name="referer" value="">
+<input type="hidden" name="method_free" value="">
+<input type="hidden" name="method_premium" value="">
+......
+```
+*/
+pub async fn resolve_link(url: &str) -> Result<String> {
+    let user_agent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:77.0) Gecko/20100101 Firefox/77.0";
+    let accept =
+        "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
+
+    // Add a few extra headers to the request in order to be less suspicious
+    let body = reqwest::Client::new()
+        .get(url)
+        .header("User-Agent", user_agent)
+        .header("Accept", accept)
+        .send()
+        .await?
+        .text()
+        .await?;
+
+    let re_link = Regex::new(r#"<input type="hidden" name="id" value="([0-9a-zA-Z]+)">"#)?;
+
+    let cap_link = match re_link.captures(&body) {
+        Some(cap) => cap,
+        None => return Err(Error::new(ErrorKind::Other, "Link not found").into()),
+    };
+
+    let id = &match cap_link.get(1) {
+        Some(id) => id.as_str(),
+        None => return Err(Error::new(ErrorKind::Other, "Link not found").into()),
+    };
+
+    let resp = reqwest::ClientBuilder::new()
+        .redirect(reqwest::redirect::Policy::none())
+        .build()?
+        .post("https://send.cm")
+        .header("User-Agent", user_agent)
+        .header("Accept", accept)
+        .form(&[
+            ("op", "download2"),
+            ("id", id),
+            ("rand", ""),
+            ("referer", ""),
+            ("method_free", ""),
+            ("method_premium", ""),
+        ])
+        .send()
+        .await?;
+
+    if resp.status().is_redirection() {
+        match resp.headers().get(reqwest::header::LOCATION) {
+            Some(location) => Ok(location.to_str()?.to_string()),
+            None => Err(Error::new(ErrorKind::Other, "Location header not found").into()),
+        }
+    } else {
+        Err(Error::new(ErrorKind::Other, "Link not found").into())
+    }
+}

src/integrations/zippy.rs

@@ -0,0 +1,72 @@
+use std::io::{Error, ErrorKind};
+
+use anyhow::Result;
+use ducc::Ducc;
+use regex::Regex;
+
+pub fn is_zippyshare_url(url: &str) -> bool {
+    Regex::new(r#"^https?://(?:www\d*\.)?zippyshare\.com/v/[0-9a-zA-Z]+/file\.html$"#)
+        .unwrap()
+        .is_match(url)
+}
+
+pub async fn resolve_link(url: &str) -> Result<String> {
+    // Regex to check if the provided url is a zippyshare download url
+    let re = Regex::new(r#"^(https?://(?:www\d*\.)?zippyshare\.com)/v/[0-9a-zA-Z]+/file\.html$"#)?;
+    if !re.is_match(url) {
+        return Err(Error::new(ErrorKind::Other, "URL is not a zippyshare url").into());
+    }
+
+    // Extract the hostname (with https:// prefix) for later
+    let host = &re.captures(url).unwrap()[1];
+
+    // Download the html body for the download page
+    let body = reqwest::get(url).await?.text().await?;
+
+    let re_script =
+        Regex::new(r#"(?ms)<script.*?>(.*getElementById\('dlbutton'\).*?)</script>"#).unwrap();
+    let re_script_start = Regex::new(r#"(?ms)<script.*?>"#).unwrap();
+
+    // Extract the script. This will end at the correct script end, but has stuff before the start
+    let cap_tmp = match re_script.captures(&body) {
+        Some(cap) => cap,
+        None => return Err(Error::new(ErrorKind::Other, "Link not found").into()),
+    };
+    let temp = &cap_tmp[1];
+
+    // Find the correct script start
+    let pos_script_start = match re_script_start.find_iter(&temp).last() {
+        Some(cap) => cap,
+        None => return Err(Error::new(ErrorKind::Other, "Link not found").into()),
+    };
+    // Cut off the beginning to get only the script contents
+    let raw_script = &temp[pos_script_start.end()..];
+
+    // Preprocess the script
+    let script = preprocess_js(raw_script);
+
+    // Calculate the link
+    let link = eval_js_link_calculation(&script)
+        .map_err(|_| Error::new(ErrorKind::Other, "Link not found: JS eval error"))?;
+
+    let url = format!("{}{}", host, link);
+    Ok(url)
+}
+
+fn preprocess_js(js_src: &str) -> String {
+    let mut processed_src = js_src
+        .replace("document.getElementById('dlbutton').href", "href")
+        .replace("document.getElementById('fimage')", "false")
+        // Fix for antiscrape 24.07.2022
+        .replace("document.getElementById('omg').getAttribute('class')", "2")
+        // Fix for antiscrape 16.08.2022
+        .replace("document.getElementById('dlbutton').omg", "omg");
+
+    processed_src.push_str(";href");
+    processed_src
+}
+
+fn eval_js_link_calculation(js_src: &str) -> ducc::Result<String> {
+    let ducc = Ducc::new();
+    ducc.exec(js_src, None, Default::default())
+}

src/main.rs

@@ -15,15 +15,15 @@ use crate::args::CLIArgs;
 use crate::clireporter::cli_print_reports;
 use crate::dlreport::{DlReport, DlReporter};
 use crate::download::{download_feedback, download_feedback_multi, http_file_info};
-use crate::zippy::is_zippyshare_url;
+use crate::integrations::{is_integrated_url, resolve_integrated_url};
 
 mod args;
 mod clireporter;
 mod dlreport;
 mod download;
 mod errors;
+mod integrations;
 mod misc;
-mod zippy;
 
 struct DlRequest {
     id: usize,
@@ -109,12 +109,17 @@ async fn download_multiple(args: CLIArgs, raw_urls: Vec<String>) -> Result<()> {
 }
 
 async fn download_job(urls: SyncQueue, reporter: UnboundedSender<DlReport>, cli_args: CLIArgs) {
-    while let Some(dlreq) = urls.lock().await.pop_front() {
+    // The mutex access must be in its own scope to ensure that the lock is dropped
+    while let Some(dlreq) = {
+        let mut urls = urls.lock().await;
+        urls.pop_front().take()
+    } {
         let reporter = DlReporter::new(dlreq.id as u32, reporter.clone());
+        report_msg!(reporter, "Downloading {}", dlreq.url);
 
         // Resolve the zippy url to the direct download url if necessary
-        let url = if is_zippyshare_url(&dlreq.url) {
+        let url = match is_integrated_url(&dlreq.url) {
-            match zippy::resolve_link(&dlreq.url).await {
+            Some(service) => match resolve_integrated_url(&dlreq.url, service).await {
                 Ok(url) => url,
                 Err(_e) => {
                     report_msg!(
@@ -124,9 +129,8 @@ async fn download_job(urls: SyncQueue, reporter: UnboundedSender<DlReport>, cli_
                     );
                     continue;
                 }
-            }
+            },
-        } else {
+            None => dlreq.url,
-            dlreq.url.to_string()
         };
 
         let info = match http_file_info(&url).await {

src/zippy.rs

@@ -1,61 +0,0 @@
-use std::io::{Error, ErrorKind};
-
-use anyhow::Result;
-use regex::Regex;
-
-pub fn is_zippyshare_url(url: &str) -> bool {
-    Regex::new(r"^https?://(?:www\d*\.)?zippyshare\.com/v/[0-9a-zA-Z]+/file\.html$")
-        .unwrap()
-        .is_match(url)
-}
-
-/*
-Updated: 07.03.2022
-Link generation code:
-- `href = $1 + ($2 % $3 + $4 % $5) + $6`
-- `$1` is always `/d/XXX` where XXX is dependent on the file
-- `$2`, `$3`, `$4` and `$5` are dynamic and randomly generated on each reload
-- `$2` is always the same as `$4`
-- `$6` is dependent on the file
-- The numbers in the calculation part ($2`, `$3`, `$4` and `$5`) are hard coded
-
-```
-document.getElementById('dlbutton').href = "/d/0Ky7p1C6/" + (186549 % 51245 + 186549 % 913) + "/some-file-name.part1.rar";
-```
-*/
-pub async fn resolve_link(url: &str) -> Result<String> {
-    // Regex to check if the provided url is a zippyshare download url
-    let re = Regex::new(r"(https://www\d*\.zippyshare\.com)")?;
-    if !re.is_match(url) {
-        return Err(Error::new(ErrorKind::Other, "URL is not a zippyshare url").into());
-    }
-
-    // Extract the hostname (with https:// prefix) for later
-    let base_host = &re.captures(url).unwrap()[0];
-
-    // Download the html body for the download page
-    let body = reqwest::get(url).await?.text().await?;
-
-    // Regex to match the javascript part of the html that generates the real download link
-    let re_link = Regex::new(
-        r#"document\.getElementById\('dlbutton'\)\.href = "(/d/.+/)" \+ \((\d+) % (\d+) \+ \d+ % (\d+)\) \+ "(.+)";"#,
-    )?;
-
-    let cap_link = match re_link.captures(&body) {
-        Some(cap) => cap,
-        None => return Err(Error::new(ErrorKind::Other, "Link not found").into()),
-    };
-
-    let url_start = &cap_link[1];
-    let url_end = &cap_link[5];
-    let n2: i32 = cap_link[2].parse()?;
-    let n3: i32 = cap_link[3].parse()?;
-    let n4 = n2;
-    let n5: i32 = cap_link[4].parse()?;
-
-    let mixed = n2 % n3 + n4 % n5;
-
-    let dl_url = format!("{}{}{}{}", &base_host, url_start, mixed, url_end);
-
-    Ok(dl_url)
-}