From aed7454c5f17c14b432ef16e2f6dcf4bd58cc6b1 Mon Sep 17 00:00:00 2001 From: Esteban Blanc Date: Mon, 25 Apr 2022 23:10:49 +0200 Subject: [PATCH] misc: Prepare for release Bump crate version Update README (MSRV and usage) --- Cargo.lock | 27 ++++++++++++----------- Cargo.toml | 6 ++--- README.md | 65 ++++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 66 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5830ffd..d451f7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -540,9 +540,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6330e8a36bd8c859f3fa6d9382911fbb7147ec39807f63b923933a247240b9ba" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" [[package]] name = "httpdate" @@ -989,9 +989,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -1570,7 +1570,7 @@ dependencies = [ [[package]] name = "suckit" -version = "0.1.2" +version = "0.2.0" dependencies = [ "chrono", "colored", @@ -1728,9 +1728,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -1743,15 +1743,16 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "0f48b6d60512a392e34dbf7fd456249fd2de3c83669ab642e021903f4015185b" dependencies = [ "bytes", "libc", "memchr", "mio", "num_cpus", + "once_cell", "pin-project-lite", "socket2", "winapi", @@ -1801,9 +1802,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" dependencies = [ "proc-macro2", "quote", @@ -1827,9 +1828,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "unicode-bidi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-normalization" diff --git a/Cargo.toml b/Cargo.toml index 6192948..e8ce80b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "suckit" -version = "0.1.2" +version = "0.2.0" edition = "2018" authors = ["Esteban \"Skallwar\" Blanc ", - "Arthur \"CohenArthur\" Cohen "] + "Arthur \"CohenArthur\" Cohen "] license = "MIT OR Apache-2.0" homepage = "https://github.com/skallwar/suckit" repository = "https://github.com/skallwar/suckit" @@ -20,7 +20,7 @@ include = [ ] [package.metadata] -msrv = "1.44.1" +msrv = "1.49.0" [lib] name = "suckit" diff --git a/README.md b/README.md index d7731a3..6d521a5 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Deps](https://deps.rs/repo/github/Skallwar/suckit/status.svg)](https://deps.rs/repo/github/Skallwar/suckit) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![License](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -![MSRV](https://img.shields.io/badge/MSRV-1.46.0-blue) +![MSRV](https://img.shields.io/badge/MSRV-1.49.0-blue) # SuckIT @@ -24,21 +24,54 @@ your disk. * [ ] Saves application state on CTRL-C for later pickup # Options - -|Option|Behavior| -|---|---| -|`-h, --help`|Displays help information| -|`-v, --verbose`|Activate Verbose output| -|`-d, --depth`|Specify the level of depth to go to when visiting the website. Default is -1 (infinity)| -|`--ext-depth`|Specify the level of depth to go to when visiting websites that have a different domain name. Default is 0 (ignore external links), -1 is infinity| -|`-j, --jobs`|Number of threads to use| -|`-o, --output`|Output directory where the downloaded files are written| -|`-t, --tries`|Number of times to retry when the downloading of a page fails| -|`-u, --user-agent`|User agent to be used for sending requests| -|`-i, --include`|Specify a regex to include pages that match this pattern| -|`-e, --exclude`|Specify a regex to exclude pages that match this pattern| -|`-a, --auth`|Provide usernames and passwords for the downloader to use| -|`--dry-run`|Do everything without saving the files to the disk| +```console +USAGE: + suckit [FLAGS] [OPTIONS] + +FLAGS: + -c, --continue-on-error Flag to enable or disable exit on error + --dry-run Do everything without saving the files to the disk + -h, --help Prints help information + -V, --version Prints version information + -v, --verbose Enable more information regarding the scraping process + --visit-filter-is-download-filter Use the dowload filter in/exclude regexes for visiting as well + +OPTIONS: + -a, --auth ... + HTTP basic authentication credentials space-separated as "username password host". Can be repeated for + multiple credentials as "u1 p1 h1 u2 p2 h2" + --delay + Add a delay in seconds between downloads to reduce the likelihood of getting banned [default: 0] + + -d, --depth + Maximum recursion depth to reach when visiting. Default is -1 (infinity) [default: -1] + + -e, --exclude-download + Regex filter to exclude saving pages that match this expression [default: $^] + + --exclude-visit + Regex filter to exclude visiting pages that match this expression [default: $^] + + --ext-depth + Maximum recursion depth to reach when visiting external domains. Default is 0. -1 means infinity [default: + 0] + -i, --include-download + Regex filter to limit to only saving pages that match this expression [default: .*] + + --include-visit + Regex filter to limit to only visiting pages that match this expression [default: .*] + + -j, --jobs Maximum number of threads to use concurrently [default: 1] + -o, --output Output directory + --random-range + Generate an extra random delay between downloads, from 0 to this number. This is added to the base delay + seconds [default: 0] + -t, --tries Maximum amount of retries on download failure [default: 20] + -u, --user-agent User agent to be used for sending requests [default: suckit] + +ARGS: + Entry point of the scraping +``` # Example