From d85abfd7e433efb8587dd8e130a73318a597520e Mon Sep 17 00:00:00 2001 From: Kurt Wheeler Date: Tue, 23 Oct 2018 16:17:41 -0400 Subject: [PATCH 1/6] Pin R devtools and all of the packages it depends on. Also pin biocLite.R. --- common/biocLite.R | 172 +++++++++++++++++++++ common/dockerfiles/Dockerfile.common_tests | 6 + common/install_devtools.R | 47 ++++++ workers/affymetrix_dependencies.R | 5 +- workers/dockerfiles/Dockerfile.affymetrix | 6 + workers/dockerfiles/Dockerfile.downloaders | 8 + workers/dockerfiles/Dockerfile.illumina | 6 + workers/dockerfiles/Dockerfile.no_op | 7 + workers/dockerfiles/Dockerfile.salmon | 5 + workers/dockerfiles/Dockerfile.smasher | 7 + workers/illumina_dependencies.R | 5 +- workers/install_affy_only.R | 5 +- workers/install_downloader_R_only.R | 5 +- workers/install_gene_convert.R | 5 +- workers/install_tximport.R | 4 +- workers/qn_dependencies.R | 5 +- 16 files changed, 271 insertions(+), 27 deletions(-) create mode 100644 common/biocLite.R create mode 100644 common/install_devtools.R diff --git a/common/biocLite.R b/common/biocLite.R new file mode 100644 index 000000000..2a192b914 --- /dev/null +++ b/common/biocLite.R @@ -0,0 +1,172 @@ +# This file was originally written by the bioconductor team and +# was downloaded from: https://bioconductor.org/biocLite.R +# We have included it here so that it cannot disappear or change. + + +## Mirrors: uncomment the following and change to your favorite CRAN mirror +## if you don't want to use the default (cran.rstudio.com). +## options("repos" = c(CRAN="https://cran.rstudio.com")) + +## Mirrors: uncomment the following and change to your favorite Bioconductor +## mirror, if you don't want to use the default (bioconductor.org) +## options("BioC_mirror" = "https://bioconductor.org") + +local({ + + vers <- getRversion() + biocVers <- tryCatch({ + BiocInstaller::biocVersion() # recent BiocInstaller + }, error=function(...) { # no / older BiocInstaller + BioC_version_associated_with_R_version <- + get(".BioC_version_associated_with_R_version", + envir=asNamespace("tools"), inherits=FALSE) + if (is.function(BioC_version_associated_with_R_version)) + BioC_version_associated_with_R_version() + else # numeric_version + BioC_version_associated_with_R_version + }) + + if (vers < "3.0") { + ## legacy; no need to change "3.0" ever + ## coordinate this message with .onAttach + txt <- strwrap("A new version of Bioconductor is available + after installing the most recent version of R; see + http://bioconductor.org/install", exdent=2) + message(paste(txt, collapse="\n")) + } else if ("package:BiocInstaller" %in% search()) { + ## messages even if already attached + tryCatch(BiocInstaller:::.onAttach(), error=function(...) NULL) + } + + if (vers > "2.13" && biocVers > "2.8") { + + if (exists("biocLite", .GlobalEnv, inherits=FALSE)) { + txt <- strwrap("There is an outdated biocLite() function in the + global environment; run 'rm(biocLite)' and try again.") + stop("\n", paste(txt, collapse="\n")) + } + + if (!suppressWarnings(require("BiocInstaller", quietly=TRUE))) { + a <- NULL + p <- file.path(Sys.getenv("HOME"), ".R", "repositories") + if (file.exists(p)) { + a <- tools:::.read_repositories(p) + if (!"BioCsoft" %in% rownames(a)) + a <- NULL + } + if (is.null(a)) { + p <- file.path(R.home("etc"), "repositories") + a <- tools:::.read_repositories(p) + } + if (!"package:utils" %in% search()) { + path <- "//bioconductor.org/biocLite.R" + txt <- sprintf("use 'source(\"https:%s\")' or + 'source(\"http:%s\")' to update 'BiocInstaller' after + library(\"utils\")", path, path) + message(paste(strwrap(txt), collapse="\n ")) + } else { + if (vers >= "3.2.2" && vers < "3.3.0") { + ## transitioning to https support; check availability + con <- file(fl <- tempfile(), "w") + sink(con, type="message") + tryCatch({ + xx <- close(file("https://bioconductor.org")) + }, error=function(e) { + message(conditionMessage(e)) + }) + sink(type="message") + close(con) + if (!length(readLines(fl))) + a["BioCsoft", "URL"] <- + sub("^http:", "https:", a["BioCsoft", "URL"]) + } + ## add a conditional for Bioc releases occuring WITHIN + ## a single R minor version. This is so that a user with a + ## version of R (whose etc/repositories file references the + ## no-longer-latest URL) and without BiocInstaller + ## will be pointed to the most recent repository suitable + ## for their version of R + if (vers >= "3.5.0") { + a["BioCsoft", "URL"] <- sub(as.character(biocVers), "3.7", + a["BioCsoft", "URL"]) + } else if (vers >= "3.4.0") { + a["BioCsoft", "URL"] <- sub(as.character(biocVers), "3.6", + a["BioCsoft", "URL"]) + } else if (vers >= "3.3.0") { + a["BioCsoft", "URL"] <- sub(as.character(biocVers), "3.4", + a["BioCsoft", "URL"]) + } else if (vers >= "3.2") { + a["BioCsoft", "URL"] <- sub(as.character(biocVers), "3.2", + a["BioCsoft", "URL"]) + } else if (vers == "3.1.1") { + ## R-3.1.1's etc/repositories file at the time of the release + ## of Bioc 3.0 pointed to the 2.14 repository, but we want + ## new installations to access the 3.0 repository + a["BioCsoft", "URL"] <- sub(as.character(biocVers), "3.0", + a["BioCsoft", "URL"]) + } else if (vers == "3.1.0") { + ## R-devel points to 2.14 repository + a["BioCsoft", "URL"] <- sub(as.character(biocVers), "2.14", + a["BioCsoft", "URL"]) + } else if (vers >= "2.15" && vers < "2.16") { + a["BioCsoft", "URL"] <- sub(as.character(biocVers), "2.11", + a["BioCsoft", "URL"]) + biocVers <- numeric_version("2.11") + } + install.packages("BiocInstaller", repos=a["BioCsoft", "URL"]) + if (!suppressWarnings(require("BiocInstaller", + quietly=TRUE))) { + path0 <- "//bioconductor.org/packages" + path <- sprintf("%s/%s/bioc", path0, as.character(biocVers)) + txt0 <- "'biocLite.R' failed to install 'BiocInstaller', + use 'install.packages(\"BiocInstaller\", + repos=\"https:%s\")' or + 'install.packages(\"BiocInstaller\", repos=\"http:%s\")'" + txt <- sprintf(txt0, path, path) + message(paste(strwrap(txt), collapse="\n ")) + } + } + } else { + ## BiocInstaller version 1.16.0-1.18.1 do not + ## automatically update when type=="source"; notify users + ## when they have updated R over their old libraries + installerVersion <- utils::packageVersion("BiocInstaller") + test0 <- (vers > "3.1.2") && + !identical(getOption("pkgType"), "source") && + (installerVersion >= "1.16.0") && + (installerVersion <= "1.16.4") + if (test0) { + if (installerVersion < "1.16.4") { + txt <- "Update BiocInstaller with + oldPkgType=options(pkgType=\"source\"); + biocLite(\"BiocInstaller\"); options(oldPkgType)" + message(paste(strwrap(txt, exdent=2), collapse="\n")) + } + if (vers >= "3.2") { + path <- "//bioconductor.org/biocLite.R" + txt <- sprintf("BiocInstaller version %s is too old for + R version %s; remove.packages(\"BiocInstaller\"), + re-start R, then source(\"https:%s\") or + source(\"http:%s\")", biocVers, vers, path, path) + warning(paste(strwrap(txt, exdent=2), collapse="\n")) + } + } + } + } else { + tryCatch({ + source("https://bioconductor.org/getBioC.R") + }, error=function(e) { + warning("https: failed (", conditionMessage(e), "), trying http", + immediate.=TRUE) + source("http://bioconductor.org/getBioC.R") + }) + biocLite <<- + function(pkgs, groupName="lite", ...) + { + if (missing(pkgs)) + biocinstall(groupName=groupName, ...) + else + biocinstall(pkgs=pkgs, groupName=groupName, ...) + } + } +}) diff --git a/common/dockerfiles/Dockerfile.common_tests b/common/dockerfiles/Dockerfile.common_tests index 344feda47..f9a81c9f1 100644 --- a/common/dockerfiles/Dockerfile.common_tests +++ b/common/dockerfiles/Dockerfile.common_tests @@ -36,6 +36,12 @@ WORKDIR /home/user ENV R_LIBS "/usr/local/lib/R/site-library" +COPY common/install_devtools.R . + +RUN Rscript install_devtools.R + +COPY common/biocLite.R . + COPY workers/install_affy_only.R . RUN Rscript install_affy_only.R diff --git a/common/install_devtools.R b/common/install_devtools.R new file mode 100644 index 000000000..be0afb81e --- /dev/null +++ b/common/install_devtools.R @@ -0,0 +1,47 @@ +# This script installs version 1.13.6 of the R package devtools. + +# Because devtools is the tool which will install specific versions of +# other R packages, it cannot be used it to install itself. However in +# order to ensure there are no dependency conflicts, it also needs to +# pin each version of its dependencies. + +# This script uses install.packages to install all of the dependencies +# for devtools, however install.packages unfortunately does not +# recursively install dependencies for the packages it +# installs. Therefore this script installs every dependency up the +# chain to devtools. It's not ideal, but it guarantees a stable +# install of devtools that won't break when a new version of devtools +# is released. + + +# Treat warnings as errors, set CRAN mirror, and set parallelization: +options(warn=2) +options(repos=structure(c(CRAN="https://cloud.r-project.org"))) +options(Ncpus=parallel::detectCores()) + + +install_package_version <- function(package_name, version) { + package_tarball <- paste0(package_name, "_", version, ".tar.gz") + package_url <- paste0("https://cran.r-project.org/src/contrib/", package_tarball) + + curl_result <- system(paste0("curl --head ", package_url), intern=TRUE) + if (grepl("404", curl_result[1])) { + package_url <- paste0("https://cran.r-project.org/src/contrib/Archive/", package_name, "/", package_tarball) + } + + install.packages(package_url) +} + +install_package_version("jsonlite", "1.5") +install_package_version("mime", "0.6") +install_package_version("curl", "3.2") +install_package_version("openssl", "1.0.2") +install_package_version("R6", "2.3.0") +install_package_version("httr", "1.3.1") +install_package_version("digest", "0.6.18") +install_package_version("memoise", "1.1.0") +install_package_version("whisker", "0.3-2") +install_package_version("rstudioapi", "0.8") +install_package_version("git2r", "0.23.0") +install_package_version("withr", "2.1.2") +install_package_version("devtools", "1.13.6") diff --git a/workers/affymetrix_dependencies.R b/workers/affymetrix_dependencies.R index 4b67b94cb..537519b8c 100644 --- a/workers/affymetrix_dependencies.R +++ b/workers/affymetrix_dependencies.R @@ -4,9 +4,6 @@ options(warn=2) options(repos=structure(c(CRAN="https://cloud.r-project.org"))) options(Ncpus=parallel::detectCores()) -# Install dev packages -install.packages("devtools") - # Use devtools::install_version() to install packages in cran. devtools::install_version('ff', version='2.2-13') devtools::install_version('XML', version='3.98-1.10') @@ -19,7 +16,7 @@ devtools::install_version('pkgconfig', version='2.0.1') # Bioconductor packages, installed by devtools::install_url() # devtools::install_url() requires biocLite.R -source('https://bioconductor.org/biocLite.R') +source('biocLite.R') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { diff --git a/workers/dockerfiles/Dockerfile.affymetrix b/workers/dockerfiles/Dockerfile.affymetrix index fa5affb78..71e04d856 100644 --- a/workers/dockerfiles/Dockerfile.affymetrix +++ b/workers/dockerfiles/Dockerfile.affymetrix @@ -42,6 +42,12 @@ WORKDIR /home/user ENV R_LIBS "/usr/local/lib/R/site-library" +COPY common/install_devtools.R . + +RUN Rscript install_devtools.R + +COPY common/biocLite.R . + COPY workers/affymetrix_dependencies.R . COPY workers/install_ensg_pkgs.R . diff --git a/workers/dockerfiles/Dockerfile.downloaders b/workers/dockerfiles/Dockerfile.downloaders index fb798e55f..c2d285b6c 100644 --- a/workers/dockerfiles/Dockerfile.downloaders +++ b/workers/dockerfiles/Dockerfile.downloaders @@ -37,7 +37,15 @@ WORKDIR /home/user RUN pip3 install --upgrade pip ENV R_LIBS "/usr/local/lib/R/site-library" + +COPY common/install_devtools.R . + +RUN Rscript install_devtools.R + +COPY common/biocLite.R . + COPY workers/install_downloader_R_only.R . + RUN Rscript install_downloader_R_only.R # Aspera will only install as the current user. diff --git a/workers/dockerfiles/Dockerfile.illumina b/workers/dockerfiles/Dockerfile.illumina index 047b28cab..103d3d89e 100644 --- a/workers/dockerfiles/Dockerfile.illumina +++ b/workers/dockerfiles/Dockerfile.illumina @@ -39,6 +39,12 @@ WORKDIR /home/user ENV R_LIBS "/usr/local/lib/R/site-library" +COPY common/install_devtools.R . + +RUN Rscript install_devtools.R + +COPY common/biocLite.R . + # These are for Illumina COPY workers/illumina_dependencies.R . RUN Rscript illumina_dependencies.R diff --git a/workers/dockerfiles/Dockerfile.no_op b/workers/dockerfiles/Dockerfile.no_op index e0dc27d2a..63ad8a7cb 100644 --- a/workers/dockerfiles/Dockerfile.no_op +++ b/workers/dockerfiles/Dockerfile.no_op @@ -40,6 +40,13 @@ WORKDIR /home/user # Noop-specific requirements ENV R_LIBS "/usr/local/lib/R/site-library" + +COPY common/install_devtools.R . + +RUN Rscript install_devtools.R + +COPY common/biocLite.R . + COPY workers/install_gene_convert.R . RUN Rscript install_gene_convert.R RUN mkdir -p gene_indexes diff --git a/workers/dockerfiles/Dockerfile.salmon b/workers/dockerfiles/Dockerfile.salmon index a13aa7b4b..78bc2ae10 100644 --- a/workers/dockerfiles/Dockerfile.salmon +++ b/workers/dockerfiles/Dockerfile.salmon @@ -45,6 +45,11 @@ RUN ln -sf `pwd`/Salmon-latest_linux_x86_64/bin/salmon /usr/local/bin/ RUN rm -f Salmon-${SALMON_VERSION}_linux_x86_64.tar.gz # End Salmon installation. +# Install R dependencies +COPY common/install_devtools.R . +RUN Rscript install_devtools.R +COPY common/biocLite.R . + # Install tximport COPY workers/install_tximport.R . RUN Rscript install_tximport.R diff --git a/workers/dockerfiles/Dockerfile.smasher b/workers/dockerfiles/Dockerfile.smasher index 497ba8ea5..ca2d6297c 100644 --- a/workers/dockerfiles/Dockerfile.smasher +++ b/workers/dockerfiles/Dockerfile.smasher @@ -37,6 +37,13 @@ WORKDIR /home/user # We need a few special packages for QN ENV R_LIBS "/usr/local/lib/R/site-library" + +COPY common/install_devtools.R . + +RUN Rscript install_devtools.R + +COPY common/biocLite.R . + COPY workers/qn_dependencies.R . RUN Rscript qn_dependencies.R # End QN-specific diff --git a/workers/illumina_dependencies.R b/workers/illumina_dependencies.R index 91bea98b2..db565f551 100644 --- a/workers/illumina_dependencies.R +++ b/workers/illumina_dependencies.R @@ -2,9 +2,6 @@ options(warn=2) options(repos=structure(c(CRAN="https://cloud.r-project.org"))) options(Ncpus=parallel::detectCores()) -# Install dev packages -install.packages("devtools") - devtools::install_version('doParallel', version='1.0.11') devtools::install_version('data.table', version='1.11.0') devtools::install_version('optparse', version='1.4.4') @@ -13,7 +10,7 @@ devtools::install_version('tidyverse', version='1.2.1') devtools::install_version('rlang', version='0.2.2') # devtools::install_url() requires biocLite.R -source('https://bioconductor.org/biocLite.R') +source('biocLite.R') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { diff --git a/workers/install_affy_only.R b/workers/install_affy_only.R index 055a7cdb6..62d2917fb 100644 --- a/workers/install_affy_only.R +++ b/workers/install_affy_only.R @@ -4,13 +4,10 @@ options(warn=2) options(repos=structure(c(CRAN="https://cloud.r-project.org"))) options(Ncpus=parallel::detectCores()) -# Install dev packages -install.packages("devtools") - # Bioconductor packages, installed by devtools::install_url() # devtools::install_url() requires biocLite.R -source('https://bioconductor.org/biocLite.R') +source('biocLite.R') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { diff --git a/workers/install_downloader_R_only.R b/workers/install_downloader_R_only.R index c0bb94e35..4a2433394 100644 --- a/workers/install_downloader_R_only.R +++ b/workers/install_downloader_R_only.R @@ -4,13 +4,10 @@ options(warn=2) options(repos=structure(c(CRAN="https://cloud.r-project.org"))) options(Ncpus=parallel::detectCores()) -# Install dev packages -install.packages("devtools") - # Bioconductor packages, installed by devtools::install_url() # devtools::install_url() requires biocLite.R -source('https://bioconductor.org/biocLite.R') +source('biocLite.R') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { diff --git a/workers/install_gene_convert.R b/workers/install_gene_convert.R index b90397975..7f9305d1d 100644 --- a/workers/install_gene_convert.R +++ b/workers/install_gene_convert.R @@ -1,8 +1,5 @@ options(repos=structure(c(CRAN="https://cloud.r-project.org"))) -# Install dev packages -install.packages("devtools") - devtools::install_version('data.table', version='1.11.0') devtools::install_version('optparse', version='1.4.4') @@ -10,7 +7,7 @@ devtools::install_version('tidyverse', version='1.2.1') devtools::install_version('rlang', version='0.2.2') # devtools::install_url() requires biocLite.R -source('https://bioconductor.org/biocLite.R') +source('biocLite.R') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { diff --git a/workers/install_tximport.R b/workers/install_tximport.R index 0ef1588b6..3a60d0b18 100644 --- a/workers/install_tximport.R +++ b/workers/install_tximport.R @@ -4,13 +4,11 @@ options(warn=2) options(Ncpus=parallel::detectCores()) options(repos=structure(c(CRAN="https://cloud.r-project.org"))) -install.packages("devtools") - devtools::install_version('optparse', version='1.4.4') devtools::install_version('rjson', version='0.2.19') devtools::install_version('readr', version='1.1.1') # devtools::install_url() requires biocLite.R -source('https://bioconductor.org/biocLite.R') +source('biocLite.R') devtools::install_url('https://bioconductor.org/packages/3.6/bioc/src/contrib/tximport_1.6.0.tar.gz') diff --git a/workers/qn_dependencies.R b/workers/qn_dependencies.R index 69e1ab32e..a5877a3a5 100644 --- a/workers/qn_dependencies.R +++ b/workers/qn_dependencies.R @@ -2,16 +2,13 @@ options(warn=2) options(repos=structure(c(CRAN="https://cloud.r-project.org"))) options(Ncpus=parallel::detectCores()) -# Install dev packages -install.packages("devtools") - devtools::install_version('doParallel', version='1.0.11') devtools::install_version('data.table', version='1.11.0') devtools::install_version('optparse', version='1.4.4') devtools::install_version('lazyeval', version='0.2.1') # devtools::install_url() requires biocLite.R -source('https://bioconductor.org/biocLite.R') +source('biocLite.R') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { From de231981b888295f69f526a7308c66fa230255f9 Mon Sep 17 00:00:00 2001 From: Kurt Wheeler Date: Tue, 23 Oct 2018 16:38:54 -0400 Subject: [PATCH 2/6] Add a comment to explain the R install_package_version function. --- common/install_devtools.R | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/install_devtools.R b/common/install_devtools.R index be0afb81e..839d3976b 100644 --- a/common/install_devtools.R +++ b/common/install_devtools.R @@ -21,6 +21,11 @@ options(Ncpus=parallel::detectCores()) install_package_version <- function(package_name, version) { + # This function install a specific version of a package. + + # However, because the most current version of a package lives in a + # different location than the older versions, we have to check where + # it can be found. package_tarball <- paste0(package_name, "_", version, ".tar.gz") package_url <- paste0("https://cran.r-project.org/src/contrib/", package_tarball) From 45dae1f05e6774d84c5ca80699a023034384e351 Mon Sep 17 00:00:00 2001 From: Kurt Wheeler Date: Wed, 24 Oct 2018 10:33:28 -0400 Subject: [PATCH 3/6] Use redirect URL for CRAN packages to avoid archiving issue. --- common/install_devtools.R | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/common/install_devtools.R b/common/install_devtools.R index 839d3976b..e26d30b9d 100644 --- a/common/install_devtools.R +++ b/common/install_devtools.R @@ -22,18 +22,7 @@ options(Ncpus=parallel::detectCores()) install_package_version <- function(package_name, version) { # This function install a specific version of a package. - - # However, because the most current version of a package lives in a - # different location than the older versions, we have to check where - # it can be found. - package_tarball <- paste0(package_name, "_", version, ".tar.gz") - package_url <- paste0("https://cran.r-project.org/src/contrib/", package_tarball) - - curl_result <- system(paste0("curl --head ", package_url), intern=TRUE) - if (grepl("404", curl_result[1])) { - package_url <- paste0("https://cran.r-project.org/src/contrib/Archive/", package_name, "/", package_tarball) - } - + package_url <- paste0("https://cran.r-project.org/package=", package_name, "&version=", version) install.packages(package_url) } From 27ff0c905478a52dd882c50a994edcf0050f5155 Mon Sep 17 00:00:00 2001 From: Kurt Wheeler Date: Wed, 24 Oct 2018 10:58:35 -0400 Subject: [PATCH 4/6] Revert devtools URL change because install.packages doesn't follow redirects. --- common/install_devtools.R | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/common/install_devtools.R b/common/install_devtools.R index e26d30b9d..839d3976b 100644 --- a/common/install_devtools.R +++ b/common/install_devtools.R @@ -22,7 +22,18 @@ options(Ncpus=parallel::detectCores()) install_package_version <- function(package_name, version) { # This function install a specific version of a package. - package_url <- paste0("https://cran.r-project.org/package=", package_name, "&version=", version) + + # However, because the most current version of a package lives in a + # different location than the older versions, we have to check where + # it can be found. + package_tarball <- paste0(package_name, "_", version, ".tar.gz") + package_url <- paste0("https://cran.r-project.org/src/contrib/", package_tarball) + + curl_result <- system(paste0("curl --head ", package_url), intern=TRUE) + if (grepl("404", curl_result[1])) { + package_url <- paste0("https://cran.r-project.org/src/contrib/Archive/", package_name, "/", package_tarball) + } + install.packages(package_url) } From d98a18f91a56564bd81f83d13d513aa09c96022d Mon Sep 17 00:00:00 2001 From: Kurt Wheeler Date: Wed, 24 Oct 2018 12:00:10 -0400 Subject: [PATCH 5/6] Exit asap if a package's version does not exist. --- common/install_devtools.R | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/install_devtools.R b/common/install_devtools.R index 839d3976b..fdbe2061d 100644 --- a/common/install_devtools.R +++ b/common/install_devtools.R @@ -32,6 +32,12 @@ install_package_version <- function(package_name, version) { curl_result <- system(paste0("curl --head ", package_url), intern=TRUE) if (grepl("404", curl_result[1])) { package_url <- paste0("https://cran.r-project.org/src/contrib/Archive/", package_name, "/", package_tarball) + + # Make sure the package actually exists in the archive! + curl_result <- system(paste0("curl --head ", package_url), intern=TRUE) + if (grepl("404", curl_result[1])) { + stop(paste("Package", package_name, "version", version, "does not exist!")) + } } install.packages(package_url) From 33dce03b6e7168e25d48f8e612fb1aae18cbb589 Mon Sep 17 00:00:00 2001 From: Kurt Wheeler Date: Thu, 25 Oct 2018 14:55:34 -0400 Subject: [PATCH 6/6] Stop using biocLite and instead install BiocInstaller via install.packages --- common/biocLite.R | 172 ------------------ common/dockerfiles/Dockerfile.common_tests | 2 - workers/affymetrix_dependencies.R | 4 +- .../processors/gene_convert.R | 4 - workers/dockerfiles/Dockerfile.affymetrix | 2 - workers/dockerfiles/Dockerfile.downloaders | 2 - workers/dockerfiles/Dockerfile.illumina | 2 - workers/dockerfiles/Dockerfile.no_op | 2 - workers/dockerfiles/Dockerfile.salmon | 1 - workers/dockerfiles/Dockerfile.smasher | 2 - workers/illumina_dependencies.R | 4 +- workers/install_affy_only.R | 4 +- workers/install_downloader_R_only.R | 4 +- workers/install_gene_convert.R | 4 +- workers/install_tximport.R | 4 +- workers/qn_dependencies.R | 4 +- 16 files changed, 14 insertions(+), 203 deletions(-) delete mode 100644 common/biocLite.R diff --git a/common/biocLite.R b/common/biocLite.R deleted file mode 100644 index 2a192b914..000000000 --- a/common/biocLite.R +++ /dev/null @@ -1,172 +0,0 @@ -# This file was originally written by the bioconductor team and -# was downloaded from: https://bioconductor.org/biocLite.R -# We have included it here so that it cannot disappear or change. - - -## Mirrors: uncomment the following and change to your favorite CRAN mirror -## if you don't want to use the default (cran.rstudio.com). -## options("repos" = c(CRAN="https://cran.rstudio.com")) - -## Mirrors: uncomment the following and change to your favorite Bioconductor -## mirror, if you don't want to use the default (bioconductor.org) -## options("BioC_mirror" = "https://bioconductor.org") - -local({ - - vers <- getRversion() - biocVers <- tryCatch({ - BiocInstaller::biocVersion() # recent BiocInstaller - }, error=function(...) { # no / older BiocInstaller - BioC_version_associated_with_R_version <- - get(".BioC_version_associated_with_R_version", - envir=asNamespace("tools"), inherits=FALSE) - if (is.function(BioC_version_associated_with_R_version)) - BioC_version_associated_with_R_version() - else # numeric_version - BioC_version_associated_with_R_version - }) - - if (vers < "3.0") { - ## legacy; no need to change "3.0" ever - ## coordinate this message with .onAttach - txt <- strwrap("A new version of Bioconductor is available - after installing the most recent version of R; see - http://bioconductor.org/install", exdent=2) - message(paste(txt, collapse="\n")) - } else if ("package:BiocInstaller" %in% search()) { - ## messages even if already attached - tryCatch(BiocInstaller:::.onAttach(), error=function(...) NULL) - } - - if (vers > "2.13" && biocVers > "2.8") { - - if (exists("biocLite", .GlobalEnv, inherits=FALSE)) { - txt <- strwrap("There is an outdated biocLite() function in the - global environment; run 'rm(biocLite)' and try again.") - stop("\n", paste(txt, collapse="\n")) - } - - if (!suppressWarnings(require("BiocInstaller", quietly=TRUE))) { - a <- NULL - p <- file.path(Sys.getenv("HOME"), ".R", "repositories") - if (file.exists(p)) { - a <- tools:::.read_repositories(p) - if (!"BioCsoft" %in% rownames(a)) - a <- NULL - } - if (is.null(a)) { - p <- file.path(R.home("etc"), "repositories") - a <- tools:::.read_repositories(p) - } - if (!"package:utils" %in% search()) { - path <- "//bioconductor.org/biocLite.R" - txt <- sprintf("use 'source(\"https:%s\")' or - 'source(\"http:%s\")' to update 'BiocInstaller' after - library(\"utils\")", path, path) - message(paste(strwrap(txt), collapse="\n ")) - } else { - if (vers >= "3.2.2" && vers < "3.3.0") { - ## transitioning to https support; check availability - con <- file(fl <- tempfile(), "w") - sink(con, type="message") - tryCatch({ - xx <- close(file("https://bioconductor.org")) - }, error=function(e) { - message(conditionMessage(e)) - }) - sink(type="message") - close(con) - if (!length(readLines(fl))) - a["BioCsoft", "URL"] <- - sub("^http:", "https:", a["BioCsoft", "URL"]) - } - ## add a conditional for Bioc releases occuring WITHIN - ## a single R minor version. This is so that a user with a - ## version of R (whose etc/repositories file references the - ## no-longer-latest URL) and without BiocInstaller - ## will be pointed to the most recent repository suitable - ## for their version of R - if (vers >= "3.5.0") { - a["BioCsoft", "URL"] <- sub(as.character(biocVers), "3.7", - a["BioCsoft", "URL"]) - } else if (vers >= "3.4.0") { - a["BioCsoft", "URL"] <- sub(as.character(biocVers), "3.6", - a["BioCsoft", "URL"]) - } else if (vers >= "3.3.0") { - a["BioCsoft", "URL"] <- sub(as.character(biocVers), "3.4", - a["BioCsoft", "URL"]) - } else if (vers >= "3.2") { - a["BioCsoft", "URL"] <- sub(as.character(biocVers), "3.2", - a["BioCsoft", "URL"]) - } else if (vers == "3.1.1") { - ## R-3.1.1's etc/repositories file at the time of the release - ## of Bioc 3.0 pointed to the 2.14 repository, but we want - ## new installations to access the 3.0 repository - a["BioCsoft", "URL"] <- sub(as.character(biocVers), "3.0", - a["BioCsoft", "URL"]) - } else if (vers == "3.1.0") { - ## R-devel points to 2.14 repository - a["BioCsoft", "URL"] <- sub(as.character(biocVers), "2.14", - a["BioCsoft", "URL"]) - } else if (vers >= "2.15" && vers < "2.16") { - a["BioCsoft", "URL"] <- sub(as.character(biocVers), "2.11", - a["BioCsoft", "URL"]) - biocVers <- numeric_version("2.11") - } - install.packages("BiocInstaller", repos=a["BioCsoft", "URL"]) - if (!suppressWarnings(require("BiocInstaller", - quietly=TRUE))) { - path0 <- "//bioconductor.org/packages" - path <- sprintf("%s/%s/bioc", path0, as.character(biocVers)) - txt0 <- "'biocLite.R' failed to install 'BiocInstaller', - use 'install.packages(\"BiocInstaller\", - repos=\"https:%s\")' or - 'install.packages(\"BiocInstaller\", repos=\"http:%s\")'" - txt <- sprintf(txt0, path, path) - message(paste(strwrap(txt), collapse="\n ")) - } - } - } else { - ## BiocInstaller version 1.16.0-1.18.1 do not - ## automatically update when type=="source"; notify users - ## when they have updated R over their old libraries - installerVersion <- utils::packageVersion("BiocInstaller") - test0 <- (vers > "3.1.2") && - !identical(getOption("pkgType"), "source") && - (installerVersion >= "1.16.0") && - (installerVersion <= "1.16.4") - if (test0) { - if (installerVersion < "1.16.4") { - txt <- "Update BiocInstaller with - oldPkgType=options(pkgType=\"source\"); - biocLite(\"BiocInstaller\"); options(oldPkgType)" - message(paste(strwrap(txt, exdent=2), collapse="\n")) - } - if (vers >= "3.2") { - path <- "//bioconductor.org/biocLite.R" - txt <- sprintf("BiocInstaller version %s is too old for - R version %s; remove.packages(\"BiocInstaller\"), - re-start R, then source(\"https:%s\") or - source(\"http:%s\")", biocVers, vers, path, path) - warning(paste(strwrap(txt, exdent=2), collapse="\n")) - } - } - } - } else { - tryCatch({ - source("https://bioconductor.org/getBioC.R") - }, error=function(e) { - warning("https: failed (", conditionMessage(e), "), trying http", - immediate.=TRUE) - source("http://bioconductor.org/getBioC.R") - }) - biocLite <<- - function(pkgs, groupName="lite", ...) - { - if (missing(pkgs)) - biocinstall(groupName=groupName, ...) - else - biocinstall(pkgs=pkgs, groupName=groupName, ...) - } - } -}) diff --git a/common/dockerfiles/Dockerfile.common_tests b/common/dockerfiles/Dockerfile.common_tests index f9a81c9f1..7f44a06a7 100644 --- a/common/dockerfiles/Dockerfile.common_tests +++ b/common/dockerfiles/Dockerfile.common_tests @@ -40,8 +40,6 @@ COPY common/install_devtools.R . RUN Rscript install_devtools.R -COPY common/biocLite.R . - COPY workers/install_affy_only.R . RUN Rscript install_affy_only.R diff --git a/workers/affymetrix_dependencies.R b/workers/affymetrix_dependencies.R index 537519b8c..20520b362 100644 --- a/workers/affymetrix_dependencies.R +++ b/workers/affymetrix_dependencies.R @@ -15,8 +15,8 @@ devtools::install_version('pkgconfig', version='2.0.1') # Bioconductor packages, installed by devtools::install_url() -# devtools::install_url() requires biocLite.R -source('biocLite.R') +# devtools::install_url() requires BiocInstaller +install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { diff --git a/workers/data_refinery_workers/processors/gene_convert.R b/workers/data_refinery_workers/processors/gene_convert.R index c8487c54d..abb50fb6d 100644 --- a/workers/data_refinery_workers/processors/gene_convert.R +++ b/workers/data_refinery_workers/processors/gene_convert.R @@ -33,10 +33,6 @@ geneIndexPath <- opt$geneIndexPath filePath <- opt$inputFile outFilePath <- opt$outputFile -# Get and use this DB -message("Loading...") -source("http://bioconductor.org/biocLite.R") - # Read the data file message("Reading data file...") suppressWarnings(data <- fread(filePath, diff --git a/workers/dockerfiles/Dockerfile.affymetrix b/workers/dockerfiles/Dockerfile.affymetrix index 71e04d856..d4b6c7ec9 100644 --- a/workers/dockerfiles/Dockerfile.affymetrix +++ b/workers/dockerfiles/Dockerfile.affymetrix @@ -46,8 +46,6 @@ COPY common/install_devtools.R . RUN Rscript install_devtools.R -COPY common/biocLite.R . - COPY workers/affymetrix_dependencies.R . COPY workers/install_ensg_pkgs.R . diff --git a/workers/dockerfiles/Dockerfile.downloaders b/workers/dockerfiles/Dockerfile.downloaders index c2d285b6c..9e7767770 100644 --- a/workers/dockerfiles/Dockerfile.downloaders +++ b/workers/dockerfiles/Dockerfile.downloaders @@ -42,8 +42,6 @@ COPY common/install_devtools.R . RUN Rscript install_devtools.R -COPY common/biocLite.R . - COPY workers/install_downloader_R_only.R . RUN Rscript install_downloader_R_only.R diff --git a/workers/dockerfiles/Dockerfile.illumina b/workers/dockerfiles/Dockerfile.illumina index 103d3d89e..9311b38e8 100644 --- a/workers/dockerfiles/Dockerfile.illumina +++ b/workers/dockerfiles/Dockerfile.illumina @@ -43,8 +43,6 @@ COPY common/install_devtools.R . RUN Rscript install_devtools.R -COPY common/biocLite.R . - # These are for Illumina COPY workers/illumina_dependencies.R . RUN Rscript illumina_dependencies.R diff --git a/workers/dockerfiles/Dockerfile.no_op b/workers/dockerfiles/Dockerfile.no_op index 63ad8a7cb..348627b86 100644 --- a/workers/dockerfiles/Dockerfile.no_op +++ b/workers/dockerfiles/Dockerfile.no_op @@ -45,8 +45,6 @@ COPY common/install_devtools.R . RUN Rscript install_devtools.R -COPY common/biocLite.R . - COPY workers/install_gene_convert.R . RUN Rscript install_gene_convert.R RUN mkdir -p gene_indexes diff --git a/workers/dockerfiles/Dockerfile.salmon b/workers/dockerfiles/Dockerfile.salmon index 78bc2ae10..effb63110 100644 --- a/workers/dockerfiles/Dockerfile.salmon +++ b/workers/dockerfiles/Dockerfile.salmon @@ -48,7 +48,6 @@ RUN rm -f Salmon-${SALMON_VERSION}_linux_x86_64.tar.gz # Install R dependencies COPY common/install_devtools.R . RUN Rscript install_devtools.R -COPY common/biocLite.R . # Install tximport COPY workers/install_tximport.R . diff --git a/workers/dockerfiles/Dockerfile.smasher b/workers/dockerfiles/Dockerfile.smasher index ca2d6297c..671b2d2c3 100644 --- a/workers/dockerfiles/Dockerfile.smasher +++ b/workers/dockerfiles/Dockerfile.smasher @@ -42,8 +42,6 @@ COPY common/install_devtools.R . RUN Rscript install_devtools.R -COPY common/biocLite.R . - COPY workers/qn_dependencies.R . RUN Rscript qn_dependencies.R # End QN-specific diff --git a/workers/illumina_dependencies.R b/workers/illumina_dependencies.R index db565f551..206ddd88b 100644 --- a/workers/illumina_dependencies.R +++ b/workers/illumina_dependencies.R @@ -9,8 +9,8 @@ devtools::install_version('lazyeval', version='0.2.1') devtools::install_version('tidyverse', version='1.2.1') devtools::install_version('rlang', version='0.2.2') -# devtools::install_url() requires biocLite.R -source('biocLite.R') +# devtools::install_url() requires BiocInstaller +install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { diff --git a/workers/install_affy_only.R b/workers/install_affy_only.R index 62d2917fb..068ecaf0b 100644 --- a/workers/install_affy_only.R +++ b/workers/install_affy_only.R @@ -6,8 +6,8 @@ options(Ncpus=parallel::detectCores()) # Bioconductor packages, installed by devtools::install_url() -# devtools::install_url() requires biocLite.R -source('biocLite.R') +# devtools::install_url() requires BiocInstaller +install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { diff --git a/workers/install_downloader_R_only.R b/workers/install_downloader_R_only.R index 4a2433394..8a635b760 100644 --- a/workers/install_downloader_R_only.R +++ b/workers/install_downloader_R_only.R @@ -6,8 +6,8 @@ options(Ncpus=parallel::detectCores()) # Bioconductor packages, installed by devtools::install_url() -# devtools::install_url() requires biocLite.R -source('biocLite.R') +# devtools::install_url() requires BiocInstaller +install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { diff --git a/workers/install_gene_convert.R b/workers/install_gene_convert.R index 7f9305d1d..a18fccead 100644 --- a/workers/install_gene_convert.R +++ b/workers/install_gene_convert.R @@ -6,8 +6,8 @@ devtools::install_version('optparse', version='1.4.4') devtools::install_version('tidyverse', version='1.2.1') devtools::install_version('rlang', version='0.2.2') -# devtools::install_url() requires biocLite.R -source('biocLite.R') +# devtools::install_url() requires BiocInstaller +install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) { diff --git a/workers/install_tximport.R b/workers/install_tximport.R index 3a60d0b18..73b36f461 100644 --- a/workers/install_tximport.R +++ b/workers/install_tximport.R @@ -8,7 +8,7 @@ devtools::install_version('optparse', version='1.4.4') devtools::install_version('rjson', version='0.2.19') devtools::install_version('readr', version='1.1.1') -# devtools::install_url() requires biocLite.R -source('biocLite.R') +# devtools::install_url() requires BiocInstaller +install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz') devtools::install_url('https://bioconductor.org/packages/3.6/bioc/src/contrib/tximport_1.6.0.tar.gz') diff --git a/workers/qn_dependencies.R b/workers/qn_dependencies.R index a5877a3a5..37487cfeb 100644 --- a/workers/qn_dependencies.R +++ b/workers/qn_dependencies.R @@ -7,8 +7,8 @@ devtools::install_version('data.table', version='1.11.0') devtools::install_version('optparse', version='1.4.4') devtools::install_version('lazyeval', version='0.2.1') -# devtools::install_url() requires biocLite.R -source('biocLite.R') +# devtools::install_url() requires BiocInstaller +install.packages('https://bioconductor.org/packages/3.6/bioc/src/contrib/BiocInstaller_1.28.0.tar.gz') # Helper function that installs a list of packages based on input URL install_with_url <- function(main_url, packages) {