feat: allo pak and cache in CI workflows

- Replace remotes with pak, which allows automatic system requirements installation - Use cache between jobs - Simplify the workflow for boodown-production - Clean bookdown workflow - Remove 'repo_name' - Allow to choose whether to update R packages or not - Add instructions in dev_history to test package gitlab-ci template manually issue #40 closes #81 closes #112 closes #100 closes #91 closes #83 closes #62 closes #55 closes #49 closes #47 closes #21
ThinkR-open · May 16, 2024 · de12dfb · de12dfb
1 parent 5d8815a
commit de12dfb
Show file tree

Hide file tree

Showing 11 changed files with 311 additions and 245 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -2,11 +2,13 @@
 
 ## Breaking changes
 
+* `use_gitlab_ci()` does not use `repo_name` anymore as "rocker" images fix CRAN to a specific date.
 * Functions deprecated since version 0.7 are removed
 * Transfer ownership of the project to ThinkR-open
 
 ## New features
 
+* `use_gitlab_ci()` allows to decide whether to update R packages during the CI pipeline
 * `gl_new_group()`, `gl_edit_group()`, `gl_delete_group()`, `gl_list_groups()`, `gl_list_sub_groups()` to deal with groups on a GitLab instance (@mpolano)
 * `gl_new_subgroup()` to create a subgroup in a group (@margotbrd)
 * `gl_delete_file()` to delete a file in a repository

diff --git a/R/ci.R b/R/ci.R
@@ -3,41 +3,46 @@
 #' @param image Docker image to use in GitLab ci. If NULL, not specified!
 #' @param path destination path for writing GitLab CI yml file
 #' @param overwrite whether to overwrite existing GitLab CI yml file
-#' @param repo_name REPO_NAME environment variable for R CRAN mirror used
 #' @param type type of the CI template to use
 #' @param add_to_Rbuildignore add CI yml file and cache path used inside the
 #' CI workflow to .Rbuildignore?
+#' @param upgrade whether to upgrade the R packages to the latest version
+#' during the CI. Default to TRUE.
 #'
 #' @details
 #' Types available are:
 #'
-#' - "check-coverage-pkgdown": Check package along with Code coverage with 'covr'
-#'  and 'pkgdown' site on GitLab Pages
+#' - "check-coverage-pkgdown": Check package along with
+#' Code coverage with 'covr' and 'pkgdown' site on GitLab Pages
 #' - "check-coverage-pkgdown-renv": Check package built in a fixed 'renv' state
 #'  along with Code coverage with 'covr' and 'pkgdown' site on GitLab Pages.
 #' - "bookdown": Build 'bookdown' HTML and PDF site on GitLab Pages
 #' - "bookdown-production": Build 'bookdown' HTML and PDF site on GitLab Pages.
-#'  Where default page is for branch named 'production' and "dev/" sub-folder is for
-#'  'main' (or 'master') branch.
+#'  Where there will be a version of the book for each branch deployed.
+#' See <https://github.com/statnmap/GitLab-Pages-Deploy> for setup details.
 #'
 #' @export
 #'
-#' @return Used for side effects. Creates a .gitlab-ci.yml file in your directory.
+#' @return Used for side effects.
+#' Creates a .gitlab-ci.yml file in your directory.
 #'
 #' @examples
 #' # Create in another directory
-#' use_gitlab_ci(image = "rocker/verse:latest", path = tempfile(fileext = ".yml"))
+#' use_gitlab_ci(
+#'   image = "rocker/verse:latest",
+#'   path = tempfile(fileext = ".yml")
+#' )
 #' \dontrun{
 #' # Create in your current project with template for packages checking
 #' use_gitlab_ci(image = "rocker/verse:latest", type = "check-coverage-pkgdown")
 #' }
 use_gitlab_ci <- function(
     image = "rocker/verse:latest",
-    repo_name = "https://packagemanager.rstudio.com/all/__linux__/focal/latest",
     path = ".gitlab-ci.yml",
     overwrite = TRUE,
     add_to_Rbuildignore = TRUE,
-    type = "check-coverage-pkgdown") {
+    type = "check-coverage-pkgdown",
+    upgrade = TRUE) {
   choices <- gsub(
     ".yml", "",
     list.files(system.file("gitlab-ci", package = "gitlabr"))
@@ -50,8 +55,8 @@ use_gitlab_ci <- function(
   lines <- readLines(file)
   # Change {image}
   lines <- gsub(pattern = "\\{image\\}", replacement = image, x = lines)
-  # Changer {repo_name}
-  lines <- gsub(pattern = "\\{repo_name\\}", replacement = repo_name, x = lines)
+  # Change {upgrade}
+  lines <- gsub(pattern = "\\{upgrade\\}", replacement = upgrade, x = lines)
 
   writeLines(enc2utf8(lines), path)
 
@@ -73,6 +78,20 @@ use_gitlab_ci <- function(
       writeLines(enc2utf8(c(r_build_ignore, "^cache$")), path_build_ignore)
     }
   }
+
+  if (type == "bookdown-production") {
+    message(
+      "You need to set up a CI/CD variable",
+      " in the GitLab project: PROJECT_ACCESS_TOKEN",
+      "\n- First, create a project access token",
+      "\n- Then, go to the project settings, CI/CD,",
+      " Variables and add the PROJECT_ACCESS_TOKEN variable",
+      "\n\nSee documentation for more details on:",
+      "https://github.com/statnmap/GitLab-Pages-Deploy"
+    )
+  }
+
+  message("GitLab CI file created at ", path)
 }
 
 #' Access the GitLab CI builds
@@ -115,8 +134,10 @@ gl_jobs <- function(project, ...) {
 #' @rdname gl_pipelines
 #' @param job Name of the job to get build artifacts from
 #' @param ref_name name of ref (i.e. branch, commit, tag). Default to 'main'.
-#' @param save_to_file either a path where to store .zip file or NULL if raw should be returned
-#' @return returns the file path if `save_to_file` is TRUE, or the archive as raw otherwise.
+#' @param save_to_file either a path where to store .zip file
+#'  or NULL if raw should be returned
+#' @return returns the file path if `save_to_file` is TRUE,
+#'  or the archive as raw otherwise.
 gl_latest_build_artifact <- function(project, job, ref_name = get_main(), save_to_file = tempfile(fileext = ".zip"), ...) {
   raw_build_archive <- gitlab(
     gl_proj_req(

diff --git a/dev/dev_history.R b/dev/dev_history.R
@@ -60,6 +60,69 @@ do.call(Sys.setenv, yaml::yaml.load_file("dev/environment.yml"))
 devtools::test() ## run all tests
 testthat::test_file("tests/testthat/test_connection_env.R") ## run test on one file
 
+# Test CI templates manually ----
+devtools::load_all()
+# >> Create a GitLab project
+set_gitlab_connection(
+  gitlab_url = test_url,
+  private_token = test_private_token,
+  api_version = test_api_version
+)
+test_pkg_gitlab <- gitlabr::gl_new_project(name = "test-pkg-ci")
+# >> Create a local package
+temp_pkg <- tempfile("test.ci.pkgfusen")
+fusen::create_fusen(
+  path = temp_pkg, template = "full",
+  open = FALSE, with_git = TRUE
+)
+usethis::with_project(temp_pkg, {
+  fusen::fill_description(
+    pkg = temp_pkg,
+    fields = list(Title = "Dummy Package")
+  )
+  usethis::use_mit_license()
+  fusen::inflate(flat_file = "dev/flat_full.Rmd", open_vignette = FALSE)
+})
+# >> Commit
+usethis::with_project(temp_pkg, {
+  # Add gitlab-ci
+  use_gitlab_ci(type = "check-coverage-pkgdown")
+  gert::git_add(".")
+  gert::git_commit_all("Add gitlab-ci")
+  gert::git_remote_add(
+    test_pkg_gitlab$http_url_to_repo,
+    name = "origin"
+  )
+})
+# >> Push
+gert::git_push(repo = temp_pkg)
+# In VSCode password prompt is on top of the window
+# >>> If push does not work directly the first time,
+# run output in Terminal:
+glue::glue("cd {temp_pkg}")
+# Then
+# git push -u origin main
+
+# >> Check the CI on GitLab - You may need to activate it manually
+
+
+# >> Modify the CI file and push again if needed
+pkgload::load_all()
+usethis::with_project(temp_pkg, {
+  # Add gitlab-ci
+  use_gitlab_ci(type = "check-coverage-pkgdown")
+  gert::git_add(".")
+  gert::git_commit_all("Add gitlab-ci again")
+  gert::git_push()
+})
+
+# Test templates are up-to-date
+testthat::test_file("tests/testthat/test_ci.R")
+# _ Delete the project
+gl_delete_project(test_pkg_gitlab$id)
+unlink(temp_pkg, recursive = TRUE)
+usethis::proj_set(path = ".", force = FALSE)
+
 # Checks for CRAN release ----
 
 ## Prepare for CRAN ----

diff --git a/inst/gitlab-ci/bookdown-production.yml b/inst/gitlab-ci/bookdown-production.yml
@@ -2,120 +2,64 @@ image: {image}
 
 variables:
   GIT_DEPTH: 10
-  REPO_NAME: "{repo_name}"
-  R_LIBS_USER: "ci/lib"
+  # R_LIBS is not empty in rocker images
+  R_LIBS: ${CI_PROJECT_DIR}/ci/lib:${R_LIBS}
+  R_LIBS_USER: ${CI_PROJECT_DIR}/ci/lib
+  SITE_BY_BRANCH: "TRUE"
 
 cache:
-  key: global-cache
+  key: cache-default
   paths:
       - ${R_LIBS_USER}
 
 stages:
   - book-create
+  - prepare-deploy
   - deploy
 
 book-branch:
     stage: book-create
     except:
-        - main
-        - master
-        - production
-        - merge_requests
+        - gh-pages
     script:
-        - lsb_release -c
-        - R -e "sessionInfo()"
-        - if [[ $CI_DEFAULT_BRANCH == "master" || $CI_DEFAULT_BRANCH = "main" ]]; then echo "OK - Default branch is master or main"; else echo "Default branch is not master or main; please add yours ($CI_DEFAULT_BRANCH) where needed, as well as in the present line of code" ; exit 1; fi
+        - echo "options(repos = c(CRAN = '${REPO_NAME}'), download.file.method = 'libcurl')" >> ${R_HOME}/etc/Rprofile.site
         - echo "Library path for packages :" $R_LIBS_USER
         - mkdir -p $R_LIBS_USER
-        - Rscript -e 'install.packages(c("remotes", "attachment", "tinytex", "usethis"))'
-        - Rscript -e 'if (!file.exists("DESCRIPTION")) {usethis::use_description()}'
+        - Rscript -e 'if(!requireNamespace("pak", quietly = TRUE)) {install.packages("pak")}'
+        - Rscript -e 'pak::pkg_install(pkg = c("attachment", "tinytex", "usethis"), upgrade = {upgrade})'
+        # - /bin/bash install_chrome.sh
+        - Rscript -e 'if (!file.exists("DESCRIPTION")) {usethis::use_description(check_name = FALSE)}'
         - >
           Rscript -e 'imports <- unique(c("bookdown", "knitr", "pagedown",
             attachment::att_from_rmds(".", recursive = FALSE)));
             if (packageVersion("attachment") <= "0.2.2") {
               attachment::att_to_desc_from_is(path.d = "DESCRIPTION", imports = imports)
               } else {attachment::att_to_desc_from_is(path.d = "DESCRIPTION", imports = imports, must.exist = FALSE)}'
-        - Rscript -e 'remotes::install_deps(dependencies = TRUE)'
+        - Rscript -e 'pak::pak(pkg = ".", upgrade = {upgrade})'
         - Rscript -e 'if(!tinytex::is_tinytex()) tinytex::install_tinytex(force = TRUE)'
         - mkdir -p public
-        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::gitbook", clean = FALSE, output_dir = "public")'
-        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::pdf_book", clean = FALSE, output_dir = "public")'
-        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "pagedown::html_paged", clean = FALSE);file.copy("_main.html", "public/_pagedown.html")'
-    artifacts:
-        paths:
-            - public
-        expire_in: 30 days
-
-
-book-main:
-    stage: book-create
-    only:
-        - master
-        - main
-        - merge_requests
-    script:
-        - echo "Library path for packages :" $R_LIBS_USER
-        - mkdir -p $R_LIBS_USER
-        - Rscript -e 'install.packages(c("remotes", "attachment", "tinytex", "usethis"))'
-        - Rscript -e 'if (!file.exists("DESCRIPTION")) {usethis::use_description()}'
-        - >
-          Rscript -e 'imports <- unique(c("bookdown", "knitr", "pagedown",
-            attachment::att_from_rmds(".", recursive = FALSE)));
-            if (packageVersion("attachment") <= "0.2.2") {
-              attachment::att_to_desc_from_is(path.d = "DESCRIPTION", imports = imports)
-              } else {attachment::att_to_desc_from_is(path.d = "DESCRIPTION", imports = imports, must.exist = FALSE)}'
-        - Rscript -e 'remotes::install_deps(dependencies = TRUE)'
-        - Rscript -e 'if(!tinytex::is_tinytex()) tinytex::install_tinytex(force = TRUE)'
-        - mkdir -p public
-        - 'curl --location --output artifacts.zip --header "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/production/download?job=pages" &&
-          unzip artifacts.zip &&
-          rm artifacts.zip &&
-          echo "copied production artifacts" ||
-          echo "copied production artifacts failed"'
-        - Rscript -e 'file.remove(list.files("public", full.names = TRUE))'
-        - Rscript -e 'unlink("public/libs", recursive = TRUE);unlink("public/images", recursive = TRUE)'
-        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::gitbook", clean = FALSE, output_dir = "public")'
-        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::pdf_book", clean = FALSE, output_dir = "public")'
-        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "pagedown::html_paged", clean = FALSE);file.copy("_main.html", "public/_pagedown.html")'
+        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::bs4_book", clean = FALSE, output_dir = "public")'
+        # - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::pdf_book", clean = FALSE, output_dir = "public")'
+        # - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "pagedown::html_paged", clean = FALSE);file.copy("_main.html", "public/index.html")'
+        # - R -e 'pagedown::chrome_print("public/index.html", "public/rapport.pdf", extra_args = c("--no-sandbox", "--disable-gpu"), verbose = TRUE)'
     artifacts:
         paths:
             - public
         expire_in: 30 days
 
-book-production:
-    stage: book-create
+gh-pages-prep:
+    stage: prepare-deploy
     only:
-        - production
+      - main
+      - master
+      - production
+      - validation
     script:
-        - echo "Library path for packages :" $R_LIBS_USER
-        - mkdir -p $R_LIBS_USER
-        - Rscript -e 'install.packages(c("remotes", "attachment", "tinytex", "usethis"))'
-        - Rscript -e 'if (!file.exists("DESCRIPTION")) {usethis::use_description()}'
-        - >
-          Rscript -e 'imports <- unique(c("bookdown", "knitr", "pagedown",
-            attachment::att_from_rmds(".", recursive = FALSE)));
-            if (packageVersion("attachment") <= "0.2.2") {
-              attachment::att_to_desc_from_is(path.d = "DESCRIPTION", imports = imports)
-              } else {attachment::att_to_desc_from_is(path.d = "DESCRIPTION", imports = imports, must.exist = FALSE)}'
-        - Rscript -e 'remotes::install_deps(dependencies = TRUE)'
-        - Rscript -e 'if(!tinytex::is_tinytex()) tinytex::install_tinytex(force = TRUE)'
-        - mkdir -p public
-        - 'curl --location --output artifacts.zip --header "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/$CI_DEFAULT_BRANCH/download?job=pages" &&
-          unzip artifacts.zip &&
-          rm artifacts.zip &&
-          echo "copied $CI_DEFAULT_BRANCH artifacts" ||
-          echo "copied $CI_DEFAULT_BRANCH artifacts failed"'
-        # Clean production directory
-        - Rscript -e 'unlink("public/production", recursive = TRUE)'
-        - mkdir -p public/production
-        # Keep clean to FALSE to retrieve intermediate files in case something fails
-        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::gitbook", clean = FALSE, output_dir = "production/dev")'
-        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::pdf_book", clean = FALSE, output_dir = "production/dev")'
-        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "pagedown::html_paged", clean = FALSE);file.copy("_main.html", "public/production/_pagedown.html")'
-    artifacts:
-        paths:
-            - public
-        expire_in: 30 days
+      # Use https://github.com/statnmap/GitLab-Pages-Deploy
+      # Deploy a unique site in gh-pages branch,
+      # or a sub-website for each branch if SITE_BY_BRANCH: "TRUE"
+      - wget https://raw.githubusercontent.com/statnmap/GitLab-Pages-Deploy/main/deploy_pages_branch.sh
+      - /bin/bash deploy_pages_branch.sh
 
 pages:
     stage: deploy
@@ -125,7 +69,6 @@ pages:
         paths:
             - public
     only:
-        - main
-        - master
-        - production
-
+        # Because we use "deploy_pages_branch", only gh-pages branch needs to be deployed
+        # All outputs from other branches in "prepare-deploy" step will push in "gh-pages"
+        - gh-pages
diff --git a/inst/gitlab-ci/bookdown.yml b/inst/gitlab-ci/bookdown.yml
@@ -2,11 +2,12 @@ image: {image}
 
 variables:
   GIT_DEPTH: 10
-  REPO_NAME: "{repo_name}"
-  R_LIBS_USER: "ci/lib"
+  # R_LIBS is not empty in rocker images
+  R_LIBS: ${CI_PROJECT_DIR}/ci/lib:${R_LIBS}
+  R_LIBS_USER: ${CI_PROJECT_DIR}/ci/lib
 
 cache:
-  key: global-cache
+  key: cache-default
   paths:
       - ${R_LIBS_USER}
 
@@ -21,17 +22,19 @@ book-main:
         - R -e "sessionInfo()"
         - echo "Library path for packages :" $R_LIBS_USER
         - mkdir -p $R_LIBS_USER
-        - Rscript -e 'install.packages(c("remotes", "attachment", "tinytex", "usethis"))'
-        - Rscript -e 'if (!file.exists("DESCRIPTION")) {usethis::use_description()}'
+        - Rscript -e 'if(!requireNamespace("pak", quietly = TRUE)) {install.packages("pak")}'
+        - Rscript -e 'pak::pkg_install(pkg = c("attachment", "tinytex", "usethis"), upgrade = {upgrade})'
+        - Rscript -e 'if (!file.exists("DESCRIPTION")) {usethis::use_description(check_name = FALSE)}'
         - >
           Rscript -e 'imports <- unique(c("bookdown", "knitr", "pagedown",
             attachment::att_from_rmds(".", recursive = FALSE)));
             if (packageVersion("attachment") <= "0.2.2") {
               attachment::att_to_desc_from_is(path.d = "DESCRIPTION", imports = imports)
               } else {attachment::att_to_desc_from_is(path.d = "DESCRIPTION", imports = imports, must.exist = FALSE)}'
-        - Rscript -e 'remotes::install_deps(dependencies = TRUE)'
+        - Rscript -e 'pak::pak(pkg = ".", upgrade = {upgrade})'
         - Rscript -e 'if(!tinytex::is_tinytex()) tinytex::install_tinytex(force = TRUE)'
         - mkdir -p public
+        - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::bs4_book", clean = FALSE, output_dir = "public")'
         - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::gitbook", clean = FALSE, output_dir = "public")'
         - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "bookdown::pdf_book", clean = FALSE, output_dir = "public")'
         - Rscript -e 'bookdown::render_book("index.Rmd", output_format = "pagedown::html_paged", clean = FALSE);file.copy("_main.html", "public/_pagedown.html")'