Merge pull request #164 from eitsupi/auto-update-core-stacks

Add scripts and workflow definitions for automatically update core dockerfiles
rocker-org · Jun 2, 2021 · e9dd8bd · e9dd8bd
2 parents 65f5249 + 3021ee3
commit e9dd8bd
Show file tree

Hide file tree

Showing 13 changed files with 354 additions and 148 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -0,0 +1,4 @@
+[*.json]
+indent_style = space
+indent_size = 2
+insert_final_newline = true
diff --git a/.github/workflows/dockerfiles.yml b/.github/workflows/dockerfiles.yml
@@ -1,19 +1,38 @@
 name: Update Dockerfiles and docker-compose files
 
-'on': [push]
-#  release:
-#    types: [published]
+on:
+  schedule:
+    - cron: "0 10 * * *"
 
 jobs:
-  build:
+  createPullRequest:
     runs-on: ubuntu-latest
+    container:
+      image: rocker/tidyverse:latest
     steps:
-      - uses: actions/checkout@v1
-      - uses: r-lib/actions/setup-r@v1
-      - name: Make Dockerfiles
-        run: make dockerfiles
-      - name: Commit back to repository
+      - uses: actions/checkout@v2
+      - name: install packages
         run: |
-          git add dockerfiles/*  compose/*
-          git commit -m "Update Dockerfiles and docker-compose files" || echo "No changes to commit"
-          git push origin ${GITHUB_REF##*/} || echo "No changes to commit"
+          install2.r --error --skipinstalled -r https://r-lib.github.io/p/pak/dev/ -n -1 pak
+      - name: Make changes to pull request
+        run: |
+          ./make-stacks.R
+          make setup
+      - name: Create Pull Request
+        id: cpr
+        uses: peter-evans/create-pull-request@v3
+        with:
+          commit-message: Automatic update of container definition files
+          signoff: false
+          branch: auto-detected-updates
+          delete-branch: true
+          title: 'Automatic update of container definition files'
+          body: |
+            - Auto-generated by [create-pull-request][1]
+
+            [1]: https://github.com/peter-evans/create-pull-request
+          draft: false
+      - name: Check outputs
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
diff --git a/dockerfiles/Dockerfile_r-ver_4.0.5 b/dockerfiles/Dockerfile_r-ver_4.0.5
@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:focal
 
 LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.source="https://github.com/rocker-org/rocker-versioned2" \

diff --git a/dockerfiles/Dockerfile_r-ver_4.1.0 b/dockerfiles/Dockerfile_r-ver_4.1.0
@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:focal
 
 LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.source="https://github.com/rocker-org/rocker-versioned2" \

diff --git a/dockerfiles/Dockerfile_r-ver_devel b/dockerfiles/Dockerfile_r-ver_devel
@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:latest
 
 LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.source="https://github.com/rocker-org/rocker-versioned2" \

diff --git a/dockerfiles/Dockerfile_rstudio_4.1.0 b/dockerfiles/Dockerfile_rstudio_4.1.0
@@ -6,7 +6,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.authors="Carl Boettiger <[email protected]>"
 
 ENV S6_VERSION=v2.1.0.2
-ENV RSTUDIO_VERSION=latest
+ENV RSTUDIO_VERSION=1.4.1717
 ENV PATH=/usr/lib/rstudio-server/bin:$PATH
 
 

diff --git a/dockerfiles/Dockerfile_rstudio_devel b/dockerfiles/Dockerfile_rstudio_devel
@@ -5,7 +5,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.vendor="Rocker Project" \
       org.opencontainers.image.authors="Carl Boettiger <[email protected]>"
 
-ENV S6_VERSION=v1.21.7.0
+ENV S6_VERSION=v2.1.0.2
 ENV RSTUDIO_VERSION=latest
 ENV PATH=/usr/lib/rstudio-server/bin:$PATH
 

diff --git a/dockerfiles/Dockerfile_verse_4.0.5 b/dockerfiles/Dockerfile_verse_4.0.5
@@ -5,7 +5,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.vendor="Rocker Project" \
       org.opencontainers.image.authors="Carl Boettiger <[email protected]>"
 
-ENV CTAN_REPO=http://mirror.ctan.org/systems/texlive/tlnet
+ENV CTAN_REPO=http://www.texlive.info/tlnet-archive/2021/05/17/tlnet
 ENV PATH=/usr/local/texlive/bin/x86_64-linux:$PATH
 
 

diff --git a/dockerfiles/Dockerfile_verse_devel b/dockerfiles/Dockerfile_verse_devel
@@ -6,7 +6,7 @@ LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \
       org.opencontainers.image.authors="Carl Boettiger <[email protected]>"
 
 ENV CTAN_REPO=http://mirror.ctan.org/systems/texlive/tlnet
-ENV PATH=/opt/texlive/bin/x86_64-linux:/usr/local/texlive/bin/x86_64-linux:$PATH
+ENV PATH=/usr/local/texlive/bin/x86_64-linux:$PATH
 
 
 RUN /rocker_scripts/install_verse.sh

diff --git a/make-stacks.R b/make-stacks.R
@@ -0,0 +1,186 @@
+#!/usr/bin/env Rscript
+
+# This script only works on Ubuntu.
+# This script needs development version of pak package (>=0.1.2.9001).
+
+library(rversions)
+library(jsonlite)
+library(pak)
+library(dplyr)
+library(readr)
+library(httr)
+library(purrr)
+library(glue)
+library(tidyr)
+library(tidyselect)
+
+
+.r_versions_data <- function(min_version) {
+    data <- rversions::r_versions() %>%
+        dplyr::mutate(
+            release_date = as.Date(date),
+            freeze_date = dplyr::lead(release_date, 1) - 1
+        ) %>%
+        dplyr::filter(readr::parse_number(version) >= min_version) %>%
+        dplyr::select(version, release_date, freeze_date) %>%
+        dplyr::rowwise() %>%
+        dplyr::mutate(ubuntu_codename = .latest_ubuntu_lts_series(release_date)) %>%
+        dplyr::ungroup()
+
+    return(data)
+}
+
+.latest_ubuntu_lts_series <- function(date) {
+    data <- utils::read.csv("/usr/share/distro-info/ubuntu.csv", stringsAsFactors = FALSE) %>%
+        dplyr::filter(
+            as.Date(release) < as.Date(date),
+            version %in% grep("LTS", version, value = TRUE)
+        ) %>%
+        utils::tail(1)
+
+    return(data$series)
+}
+
+.latest_rspm_cran_url_linux <- function(date, distro_version_name) {
+    n_retry_max <- 6
+    if (is.na(date)) {
+        url <- .make_rspm_cran_url_linux(date, distro_version_name)
+        if (.is_cran_url_available(url) != TRUE) url <- NA_character_
+    } else {
+        dates <- seq(as.Date(date), as.Date(date) - n_retry_max, by = -1)
+        for (i in seq_len(length(dates))) {
+            url <- .make_rspm_cran_url_linux(dates[i], distro_version_name)
+            if (.is_cran_url_available(url) == TRUE) break
+            url <- NA_character_
+        }
+    }
+    return(url)
+}
+
+.make_rspm_cran_url_linux <- function(date, distro_version_name) {
+    if (is.na(date)) {
+        url <- paste("https://packagemanager.rstudio.com/all/__linux__", distro_version_name, "latest", sep = "/")
+    } else {
+        url <- paste("https://packagemanager.rstudio.com/cran/__linux__", distro_version_name, date, sep = "/")
+    }
+    return(url)
+}
+
+.is_cran_url_available <- function(url) {
+    repo_data <- pak::repo_ping(cran_mirror = url, bioc = FALSE)
+    return(repo_data[repo_data$name == "CRAN", ]$ok)
+}
+
+.get_github_commit_date <- function(url) {
+    httr::GET(url, httr::add_headers(accept = "application/vnd.github.v3+json")) %>%
+        httr::content() %>%
+        purrr::pluck("commit", "committer", "date") %>%
+        as.Date()
+}
+
+rstudio_versions <- function(n_versions = 10) {
+    data <- httr::GET(
+        "https://api.github.com/repos/rstudio/rstudio/tags",
+        httr::add_headers(accept = "application/vnd.github.v3+json"),
+        query = list(per_page = n_versions)
+    ) %>%
+        httr::content() %>%
+        {
+            data.frame(
+                rstudio_version = purrr::map_chr(., "name") %>% substring(2),
+                commit_url = purrr::map_chr(., c("commit", "url"))
+            )
+        } %>%
+        dplyr::rowwise() %>%
+        dplyr::mutate(commit_date = .get_github_commit_date(commit_url)) %>%
+        dplyr::ungroup() %>%
+        dplyr::select(!commit_url) %>%
+        dplyr::arrange(commit_date)
+
+    return(data)
+}
+
+.is_rstudio_deb_url <- function(rstudio_version, ubuntu_codename) {
+    os_ver <- dplyr::case_when(
+        ubuntu_codename %in% c("bionic", "focal") ~ "bionic",
+        ubuntu_codename %in% c("xenial") ~ "xenial"
+    )
+
+    is_available <- glue::glue(
+        "https://s3.amazonaws.com/rstudio-ide-build/server/{os_ver}/amd64/rstudio-server-{rstudio_version}-amd64.deb"
+    ) %>%
+        httr::HEAD() %>%
+        httr::http_status() %>%
+        purrr::pluck("category") %>%
+        {
+            ifelse(. == "Success", TRUE, FALSE)
+        }
+
+    return(is_available)
+}
+
+.latest_ctan_url <- function(date) {
+    url <- dplyr::if_else(
+        is.na(date),
+        "http://mirror.ctan.org/systems/texlive/tlnet",
+        paste0("http://www.texlive.info/tlnet-archive/", format(date, "%Y/%m/%d"), "/tlnet")
+    )
+    return(url)
+}
+
+write_stack_core <- function(r_version, ubuntu_version, cran, rstudio_version, ctan_repo) {
+    template <- jsonlite::read_json("stacks/core-devel.json")
+
+    output_path <- paste0("stacks/core-", r_version, ".json")
+
+    template$TAG <- r_version
+    # rocker/r-ver
+    template$stack[[1]]$FROM <- paste0("ubuntu:", ubuntu_version)
+    template$stack[[1]]$ENV$R_VERSION <- r_version
+    template$stack[[1]]$ENV$CRAN <- cran
+    # rocker/rstudio
+    template$stack[[2]]$FROM <- paste0("rocker/r-ver:", r_version)
+    template$stack[[2]]$ENV$RSTUDIO_VERSION <- rstudio_version
+    # rocker/tidyverse
+    template$stack[[3]]$FROM <- paste0("rocker/rstudio:", r_version)
+    # rocker/verse
+    template$stack[[4]]$FROM <- paste0("rocker/tidyverse:", r_version)
+    template$stack[[4]]$ENV$CTAN_REPO <- ctan_repo
+
+    jsonlite::write_json(template, output_path, pretty = TRUE, auto_unbox = TRUE)
+
+    message(output_path)
+}
+
+
+df_args <- .r_versions_data(min_version = 4.0) %>%
+    dplyr::rowwise() %>%
+    dplyr::mutate(
+        cran = .latest_rspm_cran_url_linux(freeze_date, ubuntu_codename),
+    ) %>%
+    dplyr::ungroup() %>%
+    tidyr::expand_grid(rstudio_versions(n_versions = 10)) %>%
+    dplyr::filter(freeze_date > commit_date | is.na(freeze_date)) %>%
+    dplyr::rowwise() %>%
+    dplyr::filter(.is_rstudio_deb_url(rstudio_version, ubuntu_codename)) %>%
+    dplyr::ungroup() %>%
+    dplyr::group_by(version) %>%
+    dplyr::slice_tail(n = 1) %>%
+    dplyr::ungroup() %>%
+    dplyr::mutate(
+        ctan_url = .latest_ctan_url(freeze_date),
+        r_latest = dplyr::if_else(dplyr::row_number() == nrow(.), TRUE, FALSE)
+    ) %>%
+    dplyr::rename(r_version = version) %>%
+    dplyr::select(!tidyselect::ends_with("_date"))
+
+message("\nstart writing stack files.")
+
+# Write stack core files.
+devnull <- df_args %>%
+    utils::tail(2) %>%
+    apply(1, function(df) {
+        write_stack_core(df[1], df[2], df[3], df[4], df[5])
+    })
+
+message("make-stacks.R done!\n")
diff --git a/stacks/core-4.0.5.json b/stacks/core-4.0.5.json
@@ -3,51 +3,50 @@
   "TAG": "4.0.5",
   "LABEL": "org.opencontainers.image.licenses=\"GPL-2.0-or-later\" \\\n      org.opencontainers.image.source=\"https://github.com/rocker-org/rocker-versioned2\" \\\n      org.opencontainers.image.vendor=\"Rocker Project\" \\\n      org.opencontainers.image.authors=\"Carl Boettiger <[email protected]>\"",
   "stack": [
-  {
-    "IMAGE": "r-ver",
-    "FROM": "ubuntu:20.04",
-    "ENV": {
-      "R_VERSION": "4.0.5",
-      "TERM": "xterm",
-      "LC_ALL": "en_US.UTF-8",
-      "LANG": "en_US.UTF-8",
-      "R_HOME": "/usr/local/lib/R",
-      "CRAN": "https://packagemanager.rstudio.com/cran/__linux__/focal/2021-05-17",
-      "TZ": "Etc/UTC"
+    {
+      "IMAGE": "r-ver",
+      "FROM": "ubuntu:focal",
+      "ENV": {
+        "R_VERSION": "4.0.5",
+        "TERM": "xterm",
+        "LC_ALL": "en_US.UTF-8",
+        "LANG": "en_US.UTF-8",
+        "R_HOME": "/usr/local/lib/R",
+        "CRAN": "https://packagemanager.rstudio.com/cran/__linux__/focal/2021-05-17",
+        "TZ": "Etc/UTC"
+      },
+      "COPY": "scripts /rocker_scripts",
+      "RUN": "/rocker_scripts/install_R.sh",
+      "CMD": "[\"R\"]"
     },
-    "COPY": "scripts /rocker_scripts",
-    "RUN": "/rocker_scripts/install_R.sh",
-    "CMD": "[\"R\"]"
-  },
-  {
-    "IMAGE": "rstudio",
-    "FROM": "rocker/r-ver:4.0.5",
-    "ENV": {
-      "S6_VERSION": "v2.1.0.2",
-      "RSTUDIO_VERSION": "1.4.1106",
-      "PATH": "/usr/lib/rstudio-server/bin:$PATH"
+    {
+      "IMAGE": "rstudio",
+      "FROM": "rocker/r-ver:4.0.5",
+      "ENV": {
+        "S6_VERSION": "v2.1.0.2",
+        "RSTUDIO_VERSION": "1.4.1106",
+        "PATH": "/usr/lib/rstudio-server/bin:$PATH"
+      },
+      "RUN": [
+        "/rocker_scripts/install_rstudio.sh",
+        "/rocker_scripts/install_pandoc.sh"
+      ],
+      "CMD": "[\"/init\"]",
+      "EXPOSE": 8787
     },
-    "RUN": ["/rocker_scripts/install_rstudio.sh", "/rocker_scripts/install_pandoc.sh"],
-    "CMD": "[\"/init\"]",
-    "EXPOSE": 8787
-  },
-  {
-    "IMAGE": "tidyverse",
-    "FROM": "rocker/rstudio:4.0.5",
-    "RUN": "/rocker_scripts/install_tidyverse.sh"
-  },
-  {
-    "IMAGE": "verse",
-    "FROM": "rocker/tidyverse:4.0.5",
-    "ENV": {
-      "CTAN_REPO": "http://mirror.ctan.org/systems/texlive/tlnet", 
-      "PATH": "/usr/local/texlive/bin/x86_64-linux:$PATH"
+    {
+      "IMAGE": "tidyverse",
+      "FROM": "rocker/rstudio:4.0.5",
+      "RUN": "/rocker_scripts/install_tidyverse.sh"
     },
-    "RUN": "/rocker_scripts/install_verse.sh"
-  }
+    {
+      "IMAGE": "verse",
+      "FROM": "rocker/tidyverse:4.0.5",
+      "ENV": {
+        "CTAN_REPO": "http://www.texlive.info/tlnet-archive/2021/05/17/tlnet",
+        "PATH": "/usr/local/texlive/bin/x86_64-linux:$PATH"
+      },
+      "RUN": "/rocker_scripts/install_verse.sh"
+    }
   ]
 }
-
-
-
-