From a158d9b4d2cbdb109e79d00409f1f43f598201d7 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:14:45 -0400 Subject: [PATCH 1/5] Remove old Travis CI --- .travis.yml | 28 ---------------------------- _old_.travis.yml | 31 ------------------------------- 2 files changed, 59 deletions(-) delete mode 100644 .travis.yml delete mode 100644 _old_.travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ecab526..0000000 --- a/.travis.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Sample .travis.yml for R projects - -language: r -r: - - oldrel - - release - - devel -warnings_are_errors: true -sudo: false -cache: packages - -after_success: - - Rscript -e 'covr::codecov()' - -notifications: - email: - on_success: retep.meissner@gmail.com - on_failure: retep.meissner@gmail.com - -os: - - linux - -env: - matrix: - - rpkg_use_internet_for_testing=TRUE - - - diff --git a/_old_.travis.yml b/_old_.travis.yml deleted file mode 100644 index d56cd34..0000000 --- a/_old_.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -## Sample .travis.yml file for use with metacran/r-builder -## See https://github.com/metacran/r-builder for details. - -language: c -sudo: required - -before_install: - - curl -OL https://raw.githubusercontent.com/metacran/r-builder/master/pkg-build.sh - - chmod 755 pkg-build.sh - - ./pkg-build.sh bootstrap - -install: - - ./pkg-build.sh install_deps - -script: - - ./pkg-build.sh run_tests - -after_failure: - - ./pkg-build.sh dump_logs - -notifications: - email: - on_success: retep.meissner@gmail.com - on_failure: retep.meissner@gmail.com - -env: - matrix: - - RVERSION=oldrel - - RVERSION=release - - RVERSION=devel - From 7123c18aaee8810f77597138e964ff3164ff2bed Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:37:07 -0400 Subject: [PATCH 2/5] Update badges --- README.Rmd | 18 +++++++--------- README.md | 61 ++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/README.Rmd b/README.Rmd index 91cb575..c282dc6 100644 --- a/README.Rmd +++ b/README.Rmd @@ -47,17 +47,15 @@ length.cpp <- length(grep("(^\\s*$)|(^\\s*#)|(^\\s*//)", lines.cpp, value = T *lines of R code:* `r length.R`, *lines of test code:* `r length.tests` -[![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/) -[![](https://badges.ropensci.org/25_status.svg)](https://github.com/ropensci/software-review/issues/25) - - -[![cran checks](https://cranchecks.info/badges/summary/reshape)](https://cran.r-project.org/web/checks/check_results_reshape.html) -Codecov - - - - + +[![R-CMD-check](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml) +[![Peer Reviewed](https://badges.ropensci.org/25_status.svg)](https://github.com/ropensci/software-review/issues/25) +[![Total Downloads](https://cranlogs.r-pkg.org/badges/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Monthly Downloads](https://cranlogs.r-pkg.org/badges/grand-total/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Cran Checks](https://badges.cranchecks.info/summary/robotstxt.svg)](https://cran.r-project.org/web/checks/check_results_robotstxt.html) +[![Lifecycle: Stable](https://img.shields.io/badge/lifecycle-stable-brightgreen.svg)](https://lifecycle.r-lib.org/articles/stages.html#stable) + **Development version** diff --git a/README.md b/README.md index 898b0a3..c1ab3f7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ -## A ‘robots.txt’ Parser and ‘Webbot’/‘Spider’/‘Crawler’ Permissions Checker +## A ‘robots.txt’ Parser and ‘Webbot’/‘Spider’/‘Crawler’ + +Permissions Checker [![ropensci\_footer](https://raw.githubusercontent.com/ropensci/robotstxt/master/logo/github_footer.png)](https://ropensci.org) @@ -7,21 +9,24 @@ *lines of R code:* 1007, *lines of test code:* 1758 -[![Project Status: Active – The project has reached a stable, usable -state and is being actively -developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/) -[![](https://badges.ropensci.org/25_status.svg)](https://github.com/ropensci/software-review/issues/25) - - -[![cran -checks](https://cranchecks.info/badges/summary/reshape)](https://cran.r-project.org/web/checks/check_results_reshape.html) -Codecov - - + + +[![R-CMD-check](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml) +[![Peer +Reviewed](https://badges.ropensci.org/25_status.svg)](https://github.com/ropensci/software-review/issues/25) +[![Total +Downloads](https://cranlogs.r-pkg.org/badges/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Monthly +Downloads](https://cranlogs.r-pkg.org/badges/grand-total/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Cran +Checks](https://cranchecks.info/badges/summary/robotstxt)](https://cran.r-project.org/web/checks/check_results_robotstxt.html) +[![Lifecycle: +Stable](https://img.shields.io/badge/lifecycle-stable-brightgreen.svg)](https://lifecycle.r-lib.org/articles/stages.html#stable) + **Development version** -0.7.13 - 2020-08-19 / 20:39:24 +0.7.13 - 2024-08-19 / 23:13:05 **Description** @@ -122,6 +127,9 @@ paths_allowed( ) ## wikipedia.org ## [1] TRUE FALSE +``` + +``` r paths_allowed( paths = c( @@ -504,6 +512,9 @@ rt <- as.character(rt) ## [1] "# just do it - punk\n" +``` + +``` r cat(rt) ## # just do it - punk @@ -514,7 +525,7 @@ The last HTTP request is stored in an object ``` r rt_last_http$request ## Response [https://petermeissner.de/robots.txt] -## Date: 2020-09-03 19:05 +## Date: 2024-08-19 23:31 ## Status: 200 ## Content-Type: text/plain ## Size: 20 B @@ -556,7 +567,7 @@ was going on in the client-server exchange. ``` r attr(rt, "request") ## Response [https://petermeissner.de/robots.txt] -## Date: 2020-09-03 19:05 +## Date: 2024-08-19 23:31 ## Status: 200 ## Content-Type: text/plain ## Size: 20 B @@ -588,12 +599,15 @@ rt_req$request ## GET http://petermeissner.de/robots.txt ## Output: write_memory ## Options: -## * useragent: libcurl/7.64.1 r-curl/4.3 httr/1.4.1 +## * useragent: libcurl/7.81.0 r-curl/5.2.1 httr/1.4.7 ## * ssl_verifypeer: 1 ## * httpget: TRUE ## Headers: ## * Accept: application/json, text/xml, application/xml, */* -## * user-agent: R version 3.6.3 (2020-02-29) +## * user-agent: R version 4.4.1 (2024-06-14) +``` + +``` r # response headers rt_req$all_headers @@ -609,7 +623,7 @@ rt_req$all_headers ## [1] "nginx/1.10.3 (Ubuntu)" ## ## $date -## [1] "Thu, 03 Sep 2020 19:05:45 GMT" +## [1] "Mon, 19 Aug 2024 23:31:18 GMT" ## ## $`content-type` ## [1] "text/html" @@ -639,7 +653,7 @@ rt_req$all_headers ## [1] "nginx/1.10.3 (Ubuntu)" ## ## $date -## [1] "Thu, 03 Sep 2020 19:05:45 GMT" +## [1] "Mon, 19 Aug 2024 23:31:18 GMT" ## ## $`content-type` ## [1] "text/plain" @@ -648,13 +662,13 @@ rt_req$all_headers ## [1] "20" ## ## $`last-modified` -## [1] "Thu, 03 Sep 2020 15:33:01 GMT" +## [1] "Wed, 07 Dec 2022 13:34:14 GMT" ## ## $connection ## [1] "keep-alive" ## ## $etag -## [1] "\"5f510cad-14\"" +## [1] "\"63909656-14\"" ## ## $`accept-ranges` ## [1] "bytes" @@ -698,7 +712,7 @@ as.list(rt) ## ## $request ## Response [https://petermeissner.de/robots.txt] -## Date: 2020-09-03 19:05 +## Date: 2024-08-19 23:31 ## Status: 200 ## Content-Type: text/plain ## Size: 20 B @@ -716,6 +730,9 @@ the robots.txt file. paths_allowed("petermeissner.de/I_want_to_scrape_this_now", force = TRUE, verbose = TRUE) ## petermeissner.de rt_robotstxt_http_getter: force http get ## [1] TRUE +``` + +``` r paths_allowed("petermeissner.de/I_want_to_scrape_this_now",verbose = TRUE) ## petermeissner.de rt_robotstxt_http_getter: cached http get ## [1] TRUE From 6ca7b83e9000c81e0a72abe775fd0c1a34a88f94 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:37:33 -0400 Subject: [PATCH 3/5] Remove deactivated codecov --- codecov.yml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index 69cb760..0000000 --- a/codecov.yml +++ /dev/null @@ -1 +0,0 @@ -comment: false From 7679e952b891c09671d9fb7278ab729b78d6512f Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:39:37 -0400 Subject: [PATCH 4/5] Fix Cran Check badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c1ab3f7..d3d5075 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Downloads](https://cranlogs.r-pkg.org/badges/robotstxt)](https://cran.r-project. [![Monthly Downloads](https://cranlogs.r-pkg.org/badges/grand-total/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) [![Cran -Checks](https://cranchecks.info/badges/summary/robotstxt)](https://cran.r-project.org/web/checks/check_results_robotstxt.html) +Checks](https://badges.cranchecks.info/summary/robotstxt.svg)](https://cran.r-project.org/web/checks/check_results_robotstxt.html) [![Lifecycle: Stable](https://img.shields.io/badge/lifecycle-stable-brightgreen.svg)](https://lifecycle.r-lib.org/articles/stages.html#stable) From 6de2118e0fdf0e06d4c641c7c149ce3705cff52d Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:50:08 -0400 Subject: [PATCH 5/5] Fix download descriptions --- README.Rmd | 4 ++-- README.md | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.Rmd b/README.Rmd index c282dc6..c8f5666 100644 --- a/README.Rmd +++ b/README.Rmd @@ -51,8 +51,8 @@ length.cpp <- length(grep("(^\\s*$)|(^\\s*#)|(^\\s*//)", lines.cpp, value = T [![R-CMD-check](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml) [![Peer Reviewed](https://badges.ropensci.org/25_status.svg)](https://github.com/ropensci/software-review/issues/25) -[![Total Downloads](https://cranlogs.r-pkg.org/badges/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) -[![Monthly Downloads](https://cranlogs.r-pkg.org/badges/grand-total/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Monthly Downloads](https://cranlogs.r-pkg.org/badges/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Total Downloads](https://cranlogs.r-pkg.org/badges/grand-total/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) [![Cran Checks](https://badges.cranchecks.info/summary/robotstxt.svg)](https://cran.r-project.org/web/checks/check_results_robotstxt.html) [![Lifecycle: Stable](https://img.shields.io/badge/lifecycle-stable-brightgreen.svg)](https://lifecycle.r-lib.org/articles/stages.html#stable) diff --git a/README.md b/README.md index d3d5075..ae420f0 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ Permissions Checker [![R-CMD-check](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/ropensci/robotstxt/actions/workflows/R-CMD-check.yaml) [![Peer Reviewed](https://badges.ropensci.org/25_status.svg)](https://github.com/ropensci/software-review/issues/25) -[![Total -Downloads](https://cranlogs.r-pkg.org/badges/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) [![Monthly +Downloads](https://cranlogs.r-pkg.org/badges/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) +[![Total Downloads](https://cranlogs.r-pkg.org/badges/grand-total/robotstxt)](https://cran.r-project.org/web/packages/robotstxt/index.html) [![Cran Checks](https://badges.cranchecks.info/summary/robotstxt.svg)](https://cran.r-project.org/web/checks/check_results_robotstxt.html) @@ -525,7 +525,7 @@ The last HTTP request is stored in an object ``` r rt_last_http$request ## Response [https://petermeissner.de/robots.txt] -## Date: 2024-08-19 23:31 +## Date: 2024-08-19 23:48 ## Status: 200 ## Content-Type: text/plain ## Size: 20 B @@ -567,7 +567,7 @@ was going on in the client-server exchange. ``` r attr(rt, "request") ## Response [https://petermeissner.de/robots.txt] -## Date: 2024-08-19 23:31 +## Date: 2024-08-19 23:48 ## Status: 200 ## Content-Type: text/plain ## Size: 20 B @@ -623,7 +623,7 @@ rt_req$all_headers ## [1] "nginx/1.10.3 (Ubuntu)" ## ## $date -## [1] "Mon, 19 Aug 2024 23:31:18 GMT" +## [1] "Mon, 19 Aug 2024 23:48:37 GMT" ## ## $`content-type` ## [1] "text/html" @@ -653,7 +653,7 @@ rt_req$all_headers ## [1] "nginx/1.10.3 (Ubuntu)" ## ## $date -## [1] "Mon, 19 Aug 2024 23:31:18 GMT" +## [1] "Mon, 19 Aug 2024 23:48:38 GMT" ## ## $`content-type` ## [1] "text/plain" @@ -712,7 +712,7 @@ as.list(rt) ## ## $request ## Response [https://petermeissner.de/robots.txt] -## Date: 2024-08-19 23:31 +## Date: 2024-08-19 23:48 ## Status: 200 ## Content-Type: text/plain ## Size: 20 B