diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ecab526..0000000 --- a/.travis.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Sample .travis.yml for R projects - -language: r -r: - - oldrel - - release - - devel -warnings_are_errors: true -sudo: false -cache: packages - -after_success: - - Rscript -e 'covr::codecov()' - -notifications: - email: - on_success: retep.meissner@gmail.com - on_failure: retep.meissner@gmail.com - -os: - - linux - -env: - matrix: - - rpkg_use_internet_for_testing=TRUE - - - diff --git a/README.Rmd b/README.Rmd index 91cb575..c8f5666 100644 --- a/README.Rmd +++ b/README.Rmd @@ -47,17 +47,15 @@ length.cpp <- length(grep("(^\\s*$)|(^\\s*#)|(^\\s*//)", lines.cpp, value = T *lines of R code:* `r length.R`, *lines of test code:* `r length.tests` -[![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/) -[![](https://badges.ropensci.org/25_status.svg)](https://github.com/ropensci/software-review/issues/25) - - -[![cran checks](https://cranchecks.info/badges/summary/reshape)](https://cran.r-project.org/web/checks/check_results_reshape.html) -Codecov - - - - + +[![R-CMD-check](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml) +[![Peer Reviewed](https://badges.ropensci.org/25_status.svg)](https://github.com/ropensci/software-review/issues/25) +[![Monthly Downloads](https://cranlogs.r-pkg.org/badges/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Total Downloads](https://cranlogs.r-pkg.org/badges/grand-total/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Cran Checks](https://badges.cranchecks.info/summary/robotstxt.svg)](https://cran.r-project.org/web/checks/check_results_robotstxt.html) +[![Lifecycle: Stable](https://img.shields.io/badge/lifecycle-stable-brightgreen.svg)](https://lifecycle.r-lib.org/articles/stages.html#stable) + **Development version** diff --git a/README.md b/README.md index 898b0a3..ae420f0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ -## A ‘robots.txt’ Parser and ‘Webbot’/‘Spider’/‘Crawler’ Permissions Checker +## A ‘robots.txt’ Parser and ‘Webbot’/‘Spider’/‘Crawler’ + +Permissions Checker [![ropensci\_footer](https://raw.githubusercontent.com/ropensci/robotstxt/master/logo/github_footer.png)](https://ropensci.org) @@ -7,21 +9,24 @@ *lines of R code:* 1007, *lines of test code:* 1758 -[![Project Status: Active – The project has reached a stable, usable -state and is being actively -developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/) -[![](https://badges.ropensci.org/25_status.svg)](https://github.com/ropensci/software-review/issues/25) - - -[![cran -checks](https://cranchecks.info/badges/summary/reshape)](https://cran.r-project.org/web/checks/check_results_reshape.html) -Codecov - - + + +[![R-CMD-check](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml) +[![Peer +Reviewed](https://badges.ropensci.org/25_status.svg)](https://github.com/ropensci/software-review/issues/25) +[![Monthly +Downloads](https://cranlogs.r-pkg.org/badges/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Total +Downloads](https://cranlogs.r-pkg.org/badges/grand-total/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Cran +Checks](https://badges.cranchecks.info/summary/robotstxt.svg)](https://cran.r-project.org/web/checks/check_results_robotstxt.html) +[![Lifecycle: +Stable](https://img.shields.io/badge/lifecycle-stable-brightgreen.svg)](https://lifecycle.r-lib.org/articles/stages.html#stable) + **Development version** -0.7.13 - 2020-08-19 / 20:39:24 +0.7.13 - 2024-08-19 / 23:13:05 **Description** @@ -122,6 +127,9 @@ paths_allowed( ) ## wikipedia.org ## [1] TRUE FALSE +``` + +``` r paths_allowed( paths = c( @@ -504,6 +512,9 @@ rt <- as.character(rt) ## [1] "# just do it - punk\n" +``` + +``` r cat(rt) ## # just do it - punk @@ -514,7 +525,7 @@ The last HTTP request is stored in an object ``` r rt_last_http$request ## Response [https://petermeissner.de/robots.txt] -## Date: 2020-09-03 19:05 +## Date: 2024-08-19 23:48 ## Status: 200 ## Content-Type: text/plain ## Size: 20 B @@ -556,7 +567,7 @@ was going on in the client-server exchange. ``` r attr(rt, "request") ## Response [https://petermeissner.de/robots.txt] -## Date: 2020-09-03 19:05 +## Date: 2024-08-19 23:48 ## Status: 200 ## Content-Type: text/plain ## Size: 20 B @@ -588,12 +599,15 @@ rt_req$request ## GET http://petermeissner.de/robots.txt ## Output: write_memory ## Options: -## * useragent: libcurl/7.64.1 r-curl/4.3 httr/1.4.1 +## * useragent: libcurl/7.81.0 r-curl/5.2.1 httr/1.4.7 ## * ssl_verifypeer: 1 ## * httpget: TRUE ## Headers: ## * Accept: application/json, text/xml, application/xml, */* -## * user-agent: R version 3.6.3 (2020-02-29) +## * user-agent: R version 4.4.1 (2024-06-14) +``` + +``` r # response headers rt_req$all_headers @@ -609,7 +623,7 @@ rt_req$all_headers ## [1] "nginx/1.10.3 (Ubuntu)" ## ## $date -## [1] "Thu, 03 Sep 2020 19:05:45 GMT" +## [1] "Mon, 19 Aug 2024 23:48:37 GMT" ## ## $`content-type` ## [1] "text/html" @@ -639,7 +653,7 @@ rt_req$all_headers ## [1] "nginx/1.10.3 (Ubuntu)" ## ## $date -## [1] "Thu, 03 Sep 2020 19:05:45 GMT" +## [1] "Mon, 19 Aug 2024 23:48:38 GMT" ## ## $`content-type` ## [1] "text/plain" @@ -648,13 +662,13 @@ rt_req$all_headers ## [1] "20" ## ## $`last-modified` -## [1] "Thu, 03 Sep 2020 15:33:01 GMT" +## [1] "Wed, 07 Dec 2022 13:34:14 GMT" ## ## $connection ## [1] "keep-alive" ## ## $etag -## [1] "\"5f510cad-14\"" +## [1] "\"63909656-14\"" ## ## $`accept-ranges` ## [1] "bytes" @@ -698,7 +712,7 @@ as.list(rt) ## ## $request ## Response [https://petermeissner.de/robots.txt] -## Date: 2020-09-03 19:05 +## Date: 2024-08-19 23:48 ## Status: 200 ## Content-Type: text/plain ## Size: 20 B @@ -716,6 +730,9 @@ the robots.txt file. paths_allowed("petermeissner.de/I_want_to_scrape_this_now", force = TRUE, verbose = TRUE) ## petermeissner.de rt_robotstxt_http_getter: force http get ## [1] TRUE +``` + +``` r paths_allowed("petermeissner.de/I_want_to_scrape_this_now",verbose = TRUE) ## petermeissner.de rt_robotstxt_http_getter: cached http get ## [1] TRUE diff --git a/_old_.travis.yml b/_old_.travis.yml deleted file mode 100644 index d56cd34..0000000 --- a/_old_.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -## Sample .travis.yml file for use with metacran/r-builder -## See https://github.com/metacran/r-builder for details. - -language: c -sudo: required - -before_install: - - curl -OL https://raw.githubusercontent.com/metacran/r-builder/master/pkg-build.sh - - chmod 755 pkg-build.sh - - ./pkg-build.sh bootstrap - -install: - - ./pkg-build.sh install_deps - -script: - - ./pkg-build.sh run_tests - -after_failure: - - ./pkg-build.sh dump_logs - -notifications: - email: - on_success: retep.meissner@gmail.com - on_failure: retep.meissner@gmail.com - -env: - matrix: - - RVERSION=oldrel - - RVERSION=release - - RVERSION=devel - diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index 69cb760..0000000 --- a/codecov.yml +++ /dev/null @@ -1 +0,0 @@ -comment: false