diff --git a/.Rbuildignore b/.Rbuildignore index d2e3d2cc..7672ddb1 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -10,3 +10,4 @@ README\.md CITATION _pkgdown\.yml clustermq_[0-9.]+\.tar\.gz +^\.github$ diff --git a/.github/workflows/check-standard.yaml b/.github/workflows/check-standard.yaml new file mode 100644 index 00000000..63851bbb --- /dev/null +++ b/.github/workflows/check-standard.yaml @@ -0,0 +1,132 @@ +# For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag. +# https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions +on: + push: + branches-ignore: gh-pages + pull_request: + branches-ignore: gh-pages + schedule: + - cron: "0 0 * * 2" + +name: R-CMD-check + +jobs: + R-CMD-check: + runs-on: ${{ matrix.config.os }} + + name: ${{ matrix.config.os }} (${{ matrix.config.r }}) + + strategy: + fail-fast: false + matrix: + config: + - {os: windows-latest, r: 'release'} + - {os: macOS-latest, r: 'release'} + - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} + - {os: ubuntu-20.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} + + env: + R_REMOTES_NO_ERRORS_FROM_WARNINGS: true + RSPM: ${{ matrix.config.rspm }} + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v2 + + - uses: r-lib/actions/setup-r@v1 + with: + r-version: ${{ matrix.config.r }} + + - uses: r-lib/actions/setup-pandoc@v1 + + - name: Query dependencies + run: | + install.packages('remotes') + saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) + writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") + shell: Rscript {0} + + - name: Cache R packages + if: runner.os != 'Windows' + uses: actions/cache@v2 + with: + path: ${{ env.R_LIBS_USER }} + key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} + restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1- + + - name: Install system dependencies (Linux) + if: runner.os == 'Linux' + run: | + while read -r cmd + do + eval sudo $cmd + done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))') + + - name: Install system dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew update + brew install zeromq coreutils + + - name: Set up local key-based SSH + if: runner.os != 'Windows' # GHA does not allow empty passphrase on Windows + run: | + ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q + cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys + ssh-keyscan -t rsa 127.0.0.1 >> ~/.ssh/known_hosts + echo "Host 127.0.0.1" >> ~/.ssh/config + echo " IdentityFile ~/.ssh/id_rsa" >> ~/.ssh/config + echo "$(hostname) 127.0.0.1" >> ~/.hosts + chmod og-rw ~ + + - name: Install dependencies + run: | + remotes::install_deps(dependencies = TRUE) + remotes::install_cran("rcmdcheck") + shell: Rscript {0} + + - name: Install R package and add paths + if: runner.os != 'Windows' + run: | + R CMD INSTALL . + echo '.libPaths("~/work/_temp/Library")' >> ~/.Rprofile # cmq package in R + echo "$(pwd)/tests/bin" >> $GITHUB_PATH # local cmq + sed -i "1iexport PATH=$(pwd)/tests/bin:\$PATH" ~/.bashrc || true # ssh cmq + + - name: Query capabilities + if: runner.os != 'Windows' # does not recognize -e + run: | + set -x + which R + which sbatch || echo "sbatch not found" + ssh 127.0.0.1 'which R; which sbatch; echo $PATH' || true + ssh 127.0.0.1 'R --slave --no-restore -e ".libPaths()"' || true + R --slave --no-restore -e "message(clustermq:::qsys_default)" || true + ssh 127.0.0.1 'R --slave --no-restore -e "message(clustermq:::qsys_default)"' || true + + - name: make test + if: runner.os != 'Windows' + run: | + timeout 300 make test + + - name: Check + env: + _R_CHECK_CRAN_INCOMING_REMOTE_: false + run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check") + shell: Rscript {0} + + - name: Print logs if failure + if: failure() && runner.os != 'Windows' + run: | + set -x + cat ~/*.log || true + cat ~/worker.log || true + cat ~/ssh_proxy.log || true + cat clustermq.Rcheck/tests/* || true + + - name: Upload check results + if: failure() + uses: actions/upload-artifact@main + with: + name: ${{ runner.os }}-r${{ matrix.config.r }}-results + path: check diff --git a/.gitignore b/.gitignore index 340239ea..4c7bb928 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ src/*.so *.log clustermq.Rcheck src/Makevars +.github/*.html diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index be67292f..00000000 --- a/.travis.yml +++ /dev/null @@ -1,58 +0,0 @@ -language: r - -jobs: - - os: linux - r: oldrel - - os: linux - r: release - - os: linux - r: devel - - os: osx - r: oldrel - - os: osx - r: release - -warnings_are_errors: true - -cache: packages - -before_install: - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then - sed -i "1iexport PATH=$(dirname $(which R)):$(pwd)/tests/bin:\$PATH" ~/.bashrc; - sed -i "2iexport R_LIBS_USER=$R_LIBS_USER" ~/.bashrc; - sed -i "3iexport R_LIBS_SITE=$R_LIBS_SITE" ~/.bashrc; - source ~/.bashrc; - sudo apt-get install libzmq3-dev; - elif [ "$TRAVIS_OS_NAME" = "osx" ]; then - brew update; - brew install zeromq; - fi - - ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q - - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys - - ssh-keyscan -t rsa 127.0.0.1 >> ~/.ssh/known_hosts - - echo "Host 127.0.0.1 - IdentityFile ~/.ssh/id_rsa" >> ~/.ssh/config - - echo "$(hostname) 127.0.0.1" >> ~/.hosts - -before_script: - - pwd - - cat ~/.bashrc - - which R - - which sbatch || true - - ssh 127.0.0.1 'which R; which sbatch; echo $PATH' || true - - ssh 127.0.0.1 'R --no-save --no-restore -e ".libPaths()"' || true - -script: - - R CMD build . - - R CMD INSTALL *.tar.gz - - R --no-save --no-restore -e "message(clustermq:::qsys_default)" > /dev/null - - ssh 127.0.0.1 'R --no-save --no-restore -e "message(clustermq:::qsys_default)" > /dev/null' || true - - make test - - timeout 300 make check -# - timeout 120 R CMD check --as-cran *.tar.gz - -after_failure: - - cat ~/*.log - - cat ~/worker.log || true - - cat ~/ssh_proxy.log || true - - cat clustermq.Rcheck/tests/* || true diff --git a/README.md b/README.md index 0806dd8b..19bcef59 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ClusterMQ: send R function calls as cluster jobs ================================================ [![CRAN version](http://www.r-pkg.org/badges/version/clustermq)](https://cran.r-project.org/package=clustermq) -[![Build Status](https://travis-ci.org/mschubert/clustermq.svg?branch=master)](https://travis-ci.org/mschubert/clustermq) +[![Build Status](https://github.com/mschubert/clustermq/workflows/R-CMD-check/badge.svg?branch=master)](https://github.com/mschubert/clustermq/actions) [![CRAN downloads](http://cranlogs.r-pkg.org/badges/clustermq)](http://cran.rstudio.com/web/packages/clustermq/index.html) [![DOI](https://zenodo.org/badge/DOI/10.1093/bioinformatics/btz284.svg)](https://doi.org/10.1093/bioinformatics/btz284) diff --git a/vignettes/userguide.Rmd b/vignettes/userguide.Rmd index 2cd1cd87..fc109c29 100644 --- a/vignettes/userguide.Rmd +++ b/vignettes/userguide.Rmd @@ -95,7 +95,7 @@ welcome](https://github.com/mschubert/clustermq/issues/new). remotes::install_github('mschubert/clustermq', ref="develop") ``` -## Configuration +## Configuration {#configuration} Choose your preferred parallelism using: @@ -148,7 +148,7 @@ customized](#configuration), e.g. to activate [compute environments or containers](#Environments). -### SSH connector +### SSH connector {#ssh-connector} There are reasons why you might prefer to not to work on the computing cluster directly but rather on your local machine instead. @@ -237,7 +237,7 @@ Q(fx, x=1:3, const=list(y=10), n_jobs=1) ``` If a function relies on objects in its environment that are not passed as -arguments, they can be exported using the `export` argument: +arguments (including other functions), they can be exported using the `export` argument: ```{r} fx = function(x) x * 2 + y @@ -316,6 +316,45 @@ load_mtcars_example() make(my_plan, parallelism = "clustermq", jobs = 2, verbose = 4) ``` +## Options + +The various configurable options are mentioned throughout the documentation, +where applicable, however, we list all of the options here for reference. + +Options can be set by including a call to `options( = )` in your +`.Rprofile`, or by calling `options( = )` in a script or +interactively during a session. + +* `clustermq.scheduler` - One of the supported + [`clustermq` schedulers](#configuration); options are `"LOCAL"`, + `"multiprocess"`, `"multicore"`, `"lsf"`, `"sge"`, `"slurm"`, `"pbs"`, + `"Torque"`, or `"ssh"` (default is the HPC scheduler found in `$PATH`, + otherwise `"LOCAL"`) +* `clustermq.host` - The name of the node or device for constructing the + `ZeroMQ` host address (default is `Sys.info()["nodename"]`) +* `clustermq.ssh.host` - The user name and host for + [connecting to the HPC via SSH](#ssh-connector) (e.g. `user@host`); we + recommend setting up SSH keys for password-less login +* `clustermq.ssh.log` - Path for a file (on the SSH host) that will be created + and populated with logging information regarding the SSH connection + (e.g. `"~/cmq_ssh.log"`); helpful for debugging purposes +* `clustermq.ssh.timeout` - The amount of time to wait (in seconds) for a SSH + start-up connection before timing out (default is 5 seconds) +* `clustermq.worker.timeout` - The amount of time to wait (in seconds) for + master-worker communication before timing out (default is 600 seconds) +* `clustermq.error.timeout` - The amount of time to wait (in seconds), in case + of a worker error, for the remaining workers to finish their computations + and shut down cleanly (default is `min(timeout, 30)` seconds) +* `clustermq.template` - Path to a [template file](#scheduler-templates) for + submitting HPC jobs; only necessary if using your own template, otherwise + the default template will be used (default depends on + `clustermq.scheduler`) +* `clustermq.data.warning` - The threshold for the size of the common data (in + Mb) before `clustermq` throws a warning (default is 1000) +* `clustermq.defaults` - A named-list of default values for the HPC template; + this takes precedence over defaults specified in the template file + (default is an empty list (i.e. `list()`)) + ## Troubleshooting ### Debugging workers @@ -530,7 +569,7 @@ chmod u+x ~/bin/qsub SINGULARITYENV_APPEND_PATH=~/bin ``` -## Scheduler templates +## Scheduler templates {#scheduler-templates} ### LSF {#LSF}