Merge pull request #752 from AlexsLemonade/kurtwheeler/fix-r-deps

Pin R devtools, all of the packages it depends on, and biocLite
AlexsLemonade · Oct 25, 2018 · 0b9d270 · 0b9d270
2 parents e03cd37 + 33dce03
commit 0b9d270
Show file tree

Hide file tree

Showing 16 changed files with 104 additions and 38 deletions.
diff --git a/common/dockerfiles/Dockerfile.common_tests b/common/dockerfiles/Dockerfile.common_tests
@@ -36,6 +36,10 @@ WORKDIR /home/user
 
 ENV R_LIBS "/usr/local/lib/R/site-library"
 
+COPY common/install_devtools.R .
+
+RUN Rscript install_devtools.R
+
 COPY workers/install_affy_only.R .
 
 RUN Rscript install_affy_only.R

diff --git a/common/install_devtools.R b/common/install_devtools.R
@@ -0,0 +1,58 @@
+# This script installs version 1.13.6 of the R package devtools.
+
+# Because devtools is the tool which will install specific versions of
+# other R packages, it cannot be used it to install itself. However in
+# order to ensure there are no dependency conflicts, it also needs to
+# pin each version of its dependencies.
+
+# This script uses install.packages to install all of the dependencies
+# for devtools, however install.packages unfortunately does not
+# recursively install dependencies for the packages it
+# installs. Therefore this script installs every dependency up the
+# chain to devtools. It's not ideal, but it guarantees a stable
+# install of devtools that won't break when a new version of devtools
+# is released.
+
+
+# Treat warnings as errors, set CRAN mirror, and set parallelization:
+options(warn=2)
+options(repos=structure(c(CRAN="https://cloud.r-project.org")))
+options(Ncpus=parallel::detectCores())
+
+
+install_package_version <- function(package_name, version) {
+  # This function install a specific version of a package.
+
+  # However, because the most current version of a package lives in a
+  # different location than the older versions, we have to check where
+  # it can be found.
+  package_tarball <- paste0(package_name, "_", version, ".tar.gz")
+  package_url <- paste0("https://cran.r-project.org/src/contrib/", package_tarball)
+
+  curl_result <- system(paste0("curl --head ", package_url), intern=TRUE)
+  if (grepl("404", curl_result[1])) {
+    package_url <- paste0("https://cran.r-project.org/src/contrib/Archive/", package_name, "/", package_tarball)
+
+    # Make sure the package actually exists in the archive!
+    curl_result <- system(paste0("curl --head ", package_url), intern=TRUE)
+    if (grepl("404", curl_result[1])) {
+      stop(paste("Package", package_name, "version", version, "does not exist!"))
+    }
+  }
+
+  install.packages(package_url)
+}
+
+install_package_version("jsonlite", "1.5")
+install_package_version("mime", "0.6")
+install_package_version("curl", "3.2")
+install_package_version("openssl", "1.0.2")
+install_package_version("R6", "2.3.0")
+install_package_version("httr", "1.3.1")
+install_package_version("digest", "0.6.18")
+install_package_version("memoise", "1.1.0")
+install_package_version("whisker", "0.3-2")
+install_package_version("rstudioapi", "0.8")
+install_package_version("git2r", "0.23.0")
+install_package_version("withr", "2.1.2")
+install_package_version("devtools", "1.13.6")
diff --git a/workers/affymetrix_dependencies.R b/workers/affymetrix_dependencies.R
@@ -4,9 +4,6 @@ options(warn=2)
 options(repos=structure(c(CRAN="https://cloud.r-project.org")))
 options(Ncpus=parallel::detectCores())
 
-# Install dev packages
-install.packages("devtools")
-
 # Use devtools::install_version() to install packages in cran.
 devtools::install_version('ff', version='2.2-13')
 devtools::install_version('XML', version='3.98-1.10')
@@ -18,8 +15,8 @@ devtools::install_version('pkgconfig', version='2.0.1')
 
 # Bioconductor packages, installed by devtools::install_url()
 
-# devtools::install_url() requires biocLite.R
-source('https://bioconductor.org/biocLite.R')
+# devtools::install_url() requires BiocInstaller
+install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz')
 
 # Helper function that installs a list of packages based on input URL
 install_with_url <- function(main_url, packages) {

diff --git a/workers/data_refinery_workers/processors/gene_convert.R b/workers/data_refinery_workers/processors/gene_convert.R
@@ -33,10 +33,6 @@ geneIndexPath <- opt$geneIndexPath
 filePath <- opt$inputFile
 outFilePath <- opt$outputFile
 
-# Get and use this DB
-message("Loading...")
-source("http://bioconductor.org/biocLite.R")
-
 # Read the data file
 message("Reading data file...")
 suppressWarnings(data <- fread(filePath,

diff --git a/workers/dockerfiles/Dockerfile.affymetrix b/workers/dockerfiles/Dockerfile.affymetrix
@@ -42,6 +42,10 @@ WORKDIR /home/user
 
 ENV R_LIBS "/usr/local/lib/R/site-library"
 
+COPY common/install_devtools.R .
+
+RUN Rscript install_devtools.R
+
 COPY workers/affymetrix_dependencies.R .
 COPY workers/install_ensg_pkgs.R .
 

diff --git a/workers/dockerfiles/Dockerfile.downloaders b/workers/dockerfiles/Dockerfile.downloaders
@@ -37,7 +37,13 @@ WORKDIR /home/user
 RUN pip3 install --upgrade pip
 
 ENV R_LIBS "/usr/local/lib/R/site-library"
+
+COPY common/install_devtools.R .
+
+RUN Rscript install_devtools.R
+
 COPY workers/install_downloader_R_only.R .
+
 RUN Rscript install_downloader_R_only.R
 
 # Aspera will only install as the current user.

diff --git a/workers/dockerfiles/Dockerfile.illumina b/workers/dockerfiles/Dockerfile.illumina
@@ -39,6 +39,10 @@ WORKDIR /home/user
 
 ENV R_LIBS "/usr/local/lib/R/site-library"
 
+COPY common/install_devtools.R .
+
+RUN Rscript install_devtools.R
+
 # These are for Illumina
 COPY workers/illumina_dependencies.R .
 RUN Rscript illumina_dependencies.R

diff --git a/workers/dockerfiles/Dockerfile.no_op b/workers/dockerfiles/Dockerfile.no_op
@@ -40,6 +40,11 @@ WORKDIR /home/user
 
 # Noop-specific requirements
 ENV R_LIBS "/usr/local/lib/R/site-library"
+
+COPY common/install_devtools.R .
+
+RUN Rscript install_devtools.R
+
 COPY workers/install_gene_convert.R .
 RUN Rscript install_gene_convert.R
 RUN mkdir -p gene_indexes

diff --git a/workers/dockerfiles/Dockerfile.salmon b/workers/dockerfiles/Dockerfile.salmon
@@ -45,6 +45,10 @@ RUN ln -sf `pwd`/Salmon-latest_linux_x86_64/bin/salmon /usr/local/bin/
 RUN rm -f Salmon-${SALMON_VERSION}_linux_x86_64.tar.gz
 # End Salmon installation.
 
+# Install R dependencies
+COPY common/install_devtools.R .
+RUN Rscript install_devtools.R
+
 # Install tximport
 COPY workers/install_tximport.R .
 RUN Rscript install_tximport.R

diff --git a/workers/dockerfiles/Dockerfile.smasher b/workers/dockerfiles/Dockerfile.smasher
@@ -37,6 +37,11 @@ WORKDIR /home/user
 
 # We need a few special packages for QN
 ENV R_LIBS "/usr/local/lib/R/site-library"
+
+COPY common/install_devtools.R .
+
+RUN Rscript install_devtools.R
+
 COPY workers/qn_dependencies.R .
 RUN Rscript qn_dependencies.R
 # End QN-specific

diff --git a/workers/illumina_dependencies.R b/workers/illumina_dependencies.R
@@ -2,18 +2,15 @@ options(warn=2)
 options(repos=structure(c(CRAN="https://cloud.r-project.org")))
 options(Ncpus=parallel::detectCores())
 
-# Install dev packages
-install.packages("devtools")
-
 devtools::install_version('doParallel', version='1.0.11')
 devtools::install_version('data.table', version='1.11.0')
 devtools::install_version('optparse', version='1.4.4')
 devtools::install_version('lazyeval', version='0.2.1')
 devtools::install_version('tidyverse', version='1.2.1')
 devtools::install_version('rlang', version='0.2.2')
 
-# devtools::install_url() requires biocLite.R
-source('https://bioconductor.org/biocLite.R')
+# devtools::install_url() requires BiocInstaller
+install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz')
 
 # Helper function that installs a list of packages based on input URL
 install_with_url <- function(main_url, packages) {

diff --git a/workers/install_affy_only.R b/workers/install_affy_only.R
@@ -4,13 +4,10 @@ options(warn=2)
 options(repos=structure(c(CRAN="https://cloud.r-project.org")))
 options(Ncpus=parallel::detectCores())
 
-# Install dev packages
-install.packages("devtools")
-
 # Bioconductor packages, installed by devtools::install_url()
 
-# devtools::install_url() requires biocLite.R
-source('https://bioconductor.org/biocLite.R')
+# devtools::install_url() requires BiocInstaller
+install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz')
 
 # Helper function that installs a list of packages based on input URL
 install_with_url <- function(main_url, packages) {

diff --git a/workers/install_downloader_R_only.R b/workers/install_downloader_R_only.R
@@ -4,13 +4,10 @@ options(warn=2)
 options(repos=structure(c(CRAN="https://cloud.r-project.org")))
 options(Ncpus=parallel::detectCores())
 
-# Install dev packages
-install.packages("devtools")
-
 # Bioconductor packages, installed by devtools::install_url()
 
-# devtools::install_url() requires biocLite.R
-source('https://bioconductor.org/biocLite.R')
+# devtools::install_url() requires BiocInstaller
+install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz')
 
 # Helper function that installs a list of packages based on input URL
 install_with_url <- function(main_url, packages) {

diff --git a/workers/install_gene_convert.R b/workers/install_gene_convert.R
@@ -1,16 +1,13 @@
 options(repos=structure(c(CRAN="https://cloud.r-project.org")))
 
-# Install dev packages
-install.packages("devtools")
-
 devtools::install_version('data.table', version='1.11.0')
 devtools::install_version('optparse', version='1.4.4')
 
 devtools::install_version('tidyverse', version='1.2.1')
 devtools::install_version('rlang', version='0.2.2')
 
-# devtools::install_url() requires biocLite.R
-source('https://bioconductor.org/biocLite.R')
+# devtools::install_url() requires BiocInstaller
+install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz')
 
 # Helper function that installs a list of packages based on input URL
 install_with_url <- function(main_url, packages) {

diff --git a/workers/install_tximport.R b/workers/install_tximport.R
@@ -4,13 +4,11 @@ options(warn=2)
 options(Ncpus=parallel::detectCores())
 options(repos=structure(c(CRAN="https://cloud.r-project.org")))
 
-install.packages("devtools")
-
 devtools::install_version('optparse', version='1.4.4')
 devtools::install_version('rjson', version='0.2.19')
 devtools::install_version('readr', version='1.1.1')
 
-# devtools::install_url() requires biocLite.R
-source('https://bioconductor.org/biocLite.R')
+# devtools::install_url() requires BiocInstaller
+install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz')
 
 devtools::install_url('https://bioconductor.org/packages/3.6/bioc/src/contrib/tximport_1.6.0.tar.gz')
diff --git a/workers/qn_dependencies.R b/workers/qn_dependencies.R
@@ -2,16 +2,13 @@ options(warn=2)
 options(repos=structure(c(CRAN="https://cloud.r-project.org")))
 options(Ncpus=parallel::detectCores())
 
-# Install dev packages
-install.packages("devtools")
-
 devtools::install_version('doParallel', version='1.0.11')
 devtools::install_version('data.table', version='1.11.0')
 devtools::install_version('optparse', version='1.4.4')
 devtools::install_version('lazyeval', version='0.2.1')
 
-# devtools::install_url() requires biocLite.R
-source('https://bioconductor.org/biocLite.R')
+# devtools::install_url() requires BiocInstaller
+install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz')
 
 # Helper function that installs a list of packages based on input URL
 install_with_url <- function(main_url, packages) {