diff --git a/Gemfile b/Gemfile
new file mode 100644
index 0000000..8d69492
--- /dev/null
+++ b/Gemfile
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Synchronize with https://pages.github.com/versions
+ruby '>=2.7.1'
+
+gem 'github-pages', group: :jekyll_plugins
+
+if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.0.0')
+ gem 'webrick', '>= 1.6.1'
+end
\ No newline at end of file
diff --git a/Makefile b/Makefile
index ac587b8..d35f08f 100644
--- a/Makefile
+++ b/Makefile
@@ -3,37 +3,67 @@
# Settings
MAKEFILES=Makefile $(wildcard *.mk)
-JEKYLL=jekyll
-JEKYLL_VERSION=3.7.3
+JEKYLL=bundle config --local set path .vendor/bundle && bundle install && bundle update && bundle exec jekyll
PARSER=bin/markdown_ast.rb
DST=_site
+# Check Python 3 is installed and determine if it's called via python3 or python
+# (https://stackoverflow.com/a/4933395)
+PYTHON3_EXE := $(shell which python3 2>/dev/null)
+ifneq (, $(PYTHON3_EXE))
+ ifeq (,$(findstring Microsoft/WindowsApps/python3,$(subst \,/,$(PYTHON3_EXE))))
+ PYTHON := python3
+ endif
+endif
+
+ifeq (,$(PYTHON))
+ PYTHON_EXE := $(shell which python 2>/dev/null)
+ ifneq (, $(PYTHON_EXE))
+ PYTHON_VERSION_FULL := $(wordlist 2,4,$(subst ., ,$(shell python --version 2>&1)))
+ PYTHON_VERSION_MAJOR := $(word 1,${PYTHON_VERSION_FULL})
+ ifneq (3, ${PYTHON_VERSION_MAJOR})
+ $(error "Your system does not appear to have Python 3 installed.")
+ endif
+ PYTHON := python
+ else
+ $(error "Your system does not appear to have any Python installed.")
+ endif
+endif
+
+
# Controls
.PHONY : commands clean files
-.NOTPARALLEL:
-all : commands
-## commands : show all commands.
-commands :
- @grep -h -E '^##' ${MAKEFILES} | sed -e 's/## //g'
+# Default target
+.DEFAULT_GOAL := commands
-## docker-serve : use docker to build the site
-docker-serve :
- docker run --rm -it -v ${PWD}:/srv/jekyll -p 127.0.0.1:4000:4000 jekyll/jekyll:${JEKYLL_VERSION} make serve
+## I. Commands for both workshop and lesson websites
+## =================================================
-## serve : run a local server.
+## * serve : render website and run a local server
serve : lesson-md
${JEKYLL} serve
-## site : build files but do not run a server.
+## * site : build website but do not run a server
site : lesson-md
${JEKYLL} build
-# repo-check : check repository settings.
+## * docker-serve : use Docker to serve the site
+docker-serve :
+ docker pull carpentries/lesson-docker:latest
+ docker run --rm -it \
+ -v $${PWD}:/home/rstudio \
+ -p 4000:4000 \
+ -p 8787:8787 \
+ -e USERID=$$(id -u) \
+ -e GROUPID=$$(id -g) \
+ carpentries/lesson-docker:latest
+
+## * repo-check : check repository settings
repo-check :
- @bin/repo_check.py -s .
+ @${PYTHON} bin/repo_check.py -s .
-## clean : clean up junk files.
+## * clean : clean up junk files
clean :
@rm -rf ${DST}
@rm -rf .sass-cache
@@ -42,24 +72,28 @@ clean :
@find . -name '*~' -exec rm {} \;
@find . -name '*.pyc' -exec rm {} \;
-## clean-rmd : clean intermediate R files (that need to be committed to the repo).
+## * clean-rmd : clean intermediate R files (that need to be committed to the repo)
clean-rmd :
@rm -rf ${RMD_DST}
@rm -rf fig/rmd-*
-## ----------------------------------------
-## Commands specific to workshop websites.
+
+##
+## II. Commands specific to workshop websites
+## =================================================
.PHONY : workshop-check
-## workshop-check : check workshop homepage.
+## * workshop-check : check workshop homepage
workshop-check :
- @bin/workshop_check.py .
+ @${PYTHON} bin/workshop_check.py .
+
-## ----------------------------------------
-## Commands specific to lesson websites.
+##
+## III. Commands specific to lesson websites
+## =================================================
-.PHONY : lesson-check lesson-md lesson-files lesson-fixme
+.PHONY : lesson-check lesson-md lesson-files lesson-fixme install-rmd-deps
# RMarkdown files
RMD_SRC = $(wildcard _episodes_rmd/??-*.Rmd)
@@ -85,37 +119,44 @@ HTML_DST = \
$(patsubst _extras/%.md,${DST}/%/index.html,$(sort $(wildcard _extras/*.md))) \
${DST}/license/index.html
-## lesson-md : convert Rmarkdown files to markdown
+## * install-rmd-deps : Install R packages dependencies to build the RMarkdown lesson
+install-rmd-deps:
+ @${SHELL} bin/install_r_deps.sh
+
+## * lesson-md : convert Rmarkdown files to markdown
lesson-md : ${RMD_DST}
-_episodes/%.md: _episodes_rmd/%.Rmd
+_episodes/%.md: _episodes_rmd/%.Rmd install-rmd-deps
+ @mkdir -p _episodes
@bin/knit_lessons.sh $< $@
-## lesson-check : validate lesson Markdown.
+## * lesson-check : validate lesson Markdown
lesson-check : lesson-fixme
- @bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md
+ @${PYTHON} bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md
-## lesson-check-all : validate lesson Markdown, checking line lengths and trailing whitespace.
+## * lesson-check-all : validate lesson Markdown, checking line lengths and trailing whitespace
lesson-check-all :
- @bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md -l -w --permissive
+ @${PYTHON} bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md -l -w --permissive
-## unittest : run unit tests on checking tools.
+## * unittest : run unit tests on checking tools
unittest :
- @bin/test_lesson_check.py
+ @${PYTHON} bin/test_lesson_check.py
-## lesson-files : show expected names of generated files for debugging.
+## * lesson-files : show expected names of generated files for debugging
lesson-files :
@echo 'RMD_SRC:' ${RMD_SRC}
@echo 'RMD_DST:' ${RMD_DST}
@echo 'MARKDOWN_SRC:' ${MARKDOWN_SRC}
@echo 'HTML_DST:' ${HTML_DST}
-## lesson-fixme : show FIXME markers embedded in source files.
+## * lesson-fixme : show FIXME markers embedded in source files
lesson-fixme :
- @fgrep -i -n FIXME ${MARKDOWN_SRC} || true
+ @grep --fixed-strings --word-regexp --line-number --no-messages FIXME ${MARKDOWN_SRC} || true
-#-------------------------------------------------------------------------------
-# Include extra commands if available.
-#-------------------------------------------------------------------------------
+##
+## IV. Auxililary (plumbing) commands
+## =================================================
--include commands.mk
+## * commands : show all commands.
+commands :
+ @sed -n -e '/^##/s|^##[[:space:]]*||p' $(MAKEFILE_LIST)
diff --git a/README.md b/README.md
index 8dd03c3..f517ff7 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@ We'd like to ask you to familiarize yourself with our [Contribution Guide](CONTR
the [more detailed guidelines][lesson-example] on proper formatting, ways to render the lesson locally, and even
how to write new episodes.
-Please see the current list of [issues][FIXME] for ideas for contributing to this
+Please see the current list of [issues](https://github.com/carpentries-incubator/machine-learning-novice-sklearn/issues) for ideas for contributing to this
repository. For making your contribution, we use the GitHub flow, which is
nicely explained in the chapter [Contributing to a Project](http://git-scm.com/book/en/v2/GitHub-Contributing-to-a-Project) in Pro Git
by Scott Chacon.
@@ -40,12 +40,11 @@ As determined by the attendees of CarpentryConnect Manchester 2019, the proposed
### Supervised Learning
- All models, objectives:
-
- - What it is;
- - when to use it and on what type of data;
- - how to evaluate the fit, over/underfitting;
- - computational complexity
+All models, objectives:
+- What it is;
+- when to use it and on what type of data;
+- how to evaluate the fit, over/underfitting;
+- computational complexity
#### I. Regression
diff --git a/_episodes/01-introduction.md b/_episodes/01-introduction.md
index 1bc815b..6498c09 100644
--- a/_episodes/01-introduction.md
+++ b/_episodes/01-introduction.md
@@ -112,3 +112,4 @@ Many machine learning techniques will give us an answer given some input data ev
> Write your answers into the etherpad.
{: .challenge}
+{% include links.md %}
diff --git a/_episodes/02-regression.md b/_episodes/02-regression.md
index 0708f7d..0e6d111 100644
--- a/_episodes/02-regression.md
+++ b/_episodes/02-regression.md
@@ -80,6 +80,7 @@ Results of linear regression:
x_sum= 26 y_sum= 41 x_sq_sum= 168 xy_sum= 263
m= 1.5182926829268293 c= 0.30487804878048763
~~~
+{: .output}
### Testing the accuracy of a linear regression model
@@ -483,3 +484,5 @@ The process_data function gave us a choice of plotting either the logarithmic or
> Do you think its a good idea to remove these outliers from your model?
> How might you do this automatically?
{: .challenge}
+
+{% include links.md %}
diff --git a/_episodes/03-introducing-sklearn.md b/_episodes/03-introducing-sklearn.md
index 570ef89..4a67a30 100644
--- a/_episodes/03-introducing-sklearn.md
+++ b/_episodes/03-introducing-sklearn.md
@@ -84,6 +84,8 @@ Now lets replace
~~~
error = measure_error(life_expectancy, linear_data)
~~~
+{: .language-python}
+
with
@@ -281,3 +283,5 @@ To measure the error lets calculate the RMS error on both the linear and polynom
> > ![China 2001-2016 predictions](../fig/polynomial_china_overprediction.png)
> {: .solution}
{: .challenge}
+
+{% include links.md %}
\ No newline at end of file
diff --git a/_episodes/04-clustering.md b/_episodes/04-clustering.md
index b27eed3..1e5d005 100644
--- a/_episodes/04-clustering.md
+++ b/_episodes/04-clustering.md
@@ -289,8 +289,4 @@ plt.show()
> {: .solution}
{: .challenge}
-
-
-
-
-
+{% include links.md %}
\ No newline at end of file
diff --git a/_episodes/05-dimensionality-reduction.md b/_episodes/05-dimensionality-reduction.md
index 9a881e7..5034935 100644
--- a/_episodes/05-dimensionality-reduction.md
+++ b/_episodes/05-dimensionality-reduction.md
@@ -15,3 +15,9 @@ keypoints:
- "PCA is a dimensionality reduction technique"
- "TSNE is another dimensionality reduction technique"
---
+
+# Placeholder text
+
+This will hopefully pass lesson-check
+
+{% include links.md %}
\ No newline at end of file
diff --git a/_episodes/05-neural-networks.md b/_episodes/06-neural-networks.md
similarity index 98%
rename from _episodes/05-neural-networks.md
rename to _episodes/06-neural-networks.md
index 0382995..37838e0 100644
--- a/_episodes/05-neural-networks.md
+++ b/_episodes/06-neural-networks.md
@@ -322,17 +322,17 @@ Now inside the loop we can select the data by doing `data_train = data[train]` a
data_test = data[test]
labels_test = labels[test]
- ~~~
- {: .language-python}
+~~~
+{: .language-python}
- Finally we need to train the classifier with the selected training data and then score it against the test data. The scores for each set of test data should be similar.
+Finally we need to train the classifier with the selected training data and then score it against the test data. The scores for each set of test data should be similar.
- ~~~
+~~~
mlp.fit(data_train,labels_train)
print("Testing set score", mlp.score(data_test, labels_test))
- ~~~
- {: .language-python}
+~~~
+{: .language-python}
Once we've established that cross validation was ok we can go ahead and train using the entire dataset by doing `mlp.fit(data,labels)`.
@@ -383,5 +383,4 @@ Google, Microsoft, Amazon and many others now have Cloud based Application Progr
> Does it do any better/worse than Google?
{: .challenge}
-
-
+{% include links.md %}
\ No newline at end of file
diff --git a/_episodes/06-ethics.md b/_episodes/07-ethics.md
similarity index 99%
rename from _episodes/06-ethics.md
rename to _episodes/07-ethics.md
index 6a6a348..5597186 100644
--- a/_episodes/06-ethics.md
+++ b/_episodes/07-ethics.md
@@ -65,5 +65,4 @@ Some questions you might want to ask yourself (and which an ethics committee mig
> Write down your group's answers in the etherpad.
{: .challenge}
-
-
+{% include links.md %}
\ No newline at end of file
diff --git a/_episodes/07-learn-more.md b/_episodes/08-learn-more.md
similarity index 99%
rename from _episodes/07-learn-more.md
rename to _episodes/08-learn-more.md
index 20c2761..d185887 100644
--- a/_episodes/07-learn-more.md
+++ b/_episodes/08-learn-more.md
@@ -49,3 +49,4 @@ introduction to the key concepts in machine learning from Amazon.
* [Azure AI](https://azure.microsoft.com/en-gb/overview/ai-platform/) - Microsoft's Cloud based AI platform.
+{% include links.md %}
diff --git a/_extras/discuss.md b/_extras/discuss.md
index bfc33c5..49c11d0 100644
--- a/_extras/discuss.md
+++ b/_extras/discuss.md
@@ -1,6 +1,6 @@
---
title: Discussion
---
-FIXME
+
{% include links.md %}
diff --git a/aio.md b/aio.md
index 523e7dd..0ab72ea 100644
--- a/aio.md
+++ b/aio.md
@@ -1,37 +1,13 @@
---
+permalink: /aio/index.html
---
-{% include base_path.html %}
-
-
{% comment %}
-Create an anchor for every episode.
+As a maintainer, you don't need to edit this file.
+If you notice that something doesn't work, please
+open an issue: https://github.com/carpentries/styles/issues/new
{% endcomment %}
-{% for episode in site.episodes %}
-
-{% endfor %}
+
+{% include base_path.html %}
+
+{% include aio-script.md %}
diff --git a/bin/boilerplate/.travis.yml b/bin/boilerplate/.travis.yml
index 4f23be8..f6885e9 100644
--- a/bin/boilerplate/.travis.yml
+++ b/bin/boilerplate/.travis.yml
@@ -1,23 +1,57 @@
-dist: xenial # Ubuntu 16.04 (required for python 3.7)
-language: python
-python: 3.7
+# Travis CI is only used to check the lesson and is not involved in its deployment
+dist: bionic
+language: ruby
+rvm:
+ - 2.7.1
+
branches:
only:
- gh-pages
- /.*/
+
+cache:
+ apt: true
+ bundler: true
+ directories:
+ - /home/travis/.rvm/
+ - $R_LIBS_USER
+ - $HOME/.cache/pip
+
+env:
+ global:
+ - NOKOGIRI_USE_SYSTEM_LIBRARIES=true # speeds up installation of html-proofer
+ - R_LIBS_USER=~/R/Library
+ - R_LIBS_SITE=/usr/local/lib/R/site-library:/usr/lib/R/site-library
+ - R_VERSION=4.0.2
+
before_install:
- - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9
- - echo "deb https://cran.rstudio.com/bin/linux/ubuntu trusty/" | sudo tee -a /etc/apt/sources.list
- - sudo apt-get update -y
- - sudo apt-get install -y r-base
- - sudo Rscript -e "install.packages('knitr', repos = 'https://', dependencies = TRUE)"
- - sudo Rscript -e "install.packages('stringr', repos = 'https://cran.rstudio.com', dependencies = TRUE)"
- - sudo Rscript -e "install.packages('checkpoint', repos = 'https://cran.rstudio.com', dependencies = TRUE)"
- - sudo Rscript -e "install.packages('ggplot2', repos = 'https://cran.rstudio.com', dependencies = TRUE)"
- - rvm default
- - gem install json kramdown jekyll
-install:
- - pip install pyyaml
+ ## Install R + pandoc + dependencies
+ - sudo add-apt-repository -y "ppa:marutter/rrutter4.0"
+ - sudo add-apt-repository -y "ppa:c2d4u.team/c2d4u4.0+"
+ - sudo add-apt-repository -y "ppa:ubuntugis/ppa"
+ - sudo add-apt-repository -y "ppa:cran/travis"
+ - travis_apt_get_update
+ - sudo apt-get install -y --no-install-recommends build-essential gcc g++ libblas-dev liblapack-dev libncurses5-dev libreadline-dev libjpeg-dev libpcre3-dev libpng-dev zlib1g-dev libbz2-dev liblzma-dev libicu-dev cdbs qpdf texinfo libssh2-1-dev gfortran jq python3.5 python3-pip r-base
+ - export PATH=${TRAVIS_HOME}/R-bin/bin:$PATH
+ - export LD_LIBRARY_PATH=${TRAVIS_HOME}/R-bin/lib:$LD_LIBRARY_PATH
+ - sudo mkdir -p /usr/local/lib/R/site-library $R_LIBS_USER
+ - sudo chmod 2777 /usr/local/lib/R /usr/local/lib/R/site-library $R_LIBS_USER
+ - echo 'options(repos = c(CRAN = "https://packagemanager.rstudio.com/all/__linux__/bionic/latest"))' > ~/.Rprofile.site
+ - export R_PROFILE=~/.Rprofile.site
+ - curl -fLo /tmp/texlive.tar.gz https://github.com/jimhester/ubuntu-bin/releases/download/latest/texlive.tar.gz
+ - tar xzf /tmp/texlive.tar.gz -C ~
+ - export PATH=${TRAVIS_HOME}/texlive/bin/x86_64-linux:$PATH
+ - tlmgr update --self
+ - curl -fLo /tmp/pandoc-2.2-1-amd64.deb https://github.com/jgm/pandoc/releases/download/2.2/pandoc-2.2-1-amd64.deb
+ - sudo dpkg -i /tmp/pandoc-2.2-1-amd64.deb
+ - sudo apt-get install -f
+ - rm /tmp/pandoc-2.2-1-amd64.deb
+ - Rscript -e "install.packages(setdiff(c('renv', 'rprojroot'), installed.packages()), loc = Sys.getenv('R_LIBS_USER')); update.packages(lib.loc = Sys.getenv('R_LIBS_USER'), ask = FALSE, checkBuilt = TRUE)"
+ - Rscript -e 'sessionInfo()'
+ ## Install python and dependencies
+ - python3 -m pip install --upgrade pip setuptools wheel
+ - python3 -m pip install pyyaml
+
script:
- make lesson-check-all
- make --always-make site
diff --git a/bin/boilerplate/README.md b/bin/boilerplate/README.md
index ed47f43..060994a 100644
--- a/bin/boilerplate/README.md
+++ b/bin/boilerplate/README.md
@@ -17,7 +17,7 @@ Please see the current list of [issues][FIXME] for ideas for contributing to thi
repository. For making your contribution, we use the GitHub flow, which is
nicely explained in the chapter [Contributing to a Project](http://git-scm.com/book/en/v2/GitHub-Contributing-to-a-Project) in Pro Git
by Scott Chacon.
-Look for the tag ![good_first_issue](https://img.shields.io/badge/-good%20first%20issue-gold.svg). This indicates that the mantainers will welcome a pull request fixing this issue.
+Look for the tag ![good_first_issue](https://img.shields.io/badge/-good%20first%20issue-gold.svg). This indicates that the maintainers will welcome a pull request fixing this issue.
## Maintainer(s)
diff --git a/bin/boilerplate/_config.yml b/bin/boilerplate/_config.yml
index 3c3f4f2..795f788 100644
--- a/bin/boilerplate/_config.yml
+++ b/bin/boilerplate/_config.yml
@@ -13,7 +13,8 @@ carpentry: "swc"
title: "Lesson Title"
# Life cycle stage of the lesson
-# possible values: "pre-alpha", "alpha", "beta", "stable"
+# See this page for more details: https://cdh.carpentries.org/the-lesson-life-cycle.html
+# Possible values: "pre-alpha", "alpha", "beta", "stable"
life_cycle: "pre-alpha"
#------------------------------------------------------------
@@ -32,11 +33,11 @@ repository: /
email: "team@carpentries.org"
# Sites.
-amy_site: "https://amy.software-carpentry.org/workshops"
+amy_site: "https://amy.carpentries.org/"
carpentries_github: "https://github.com/carpentries"
carpentries_pages: "https://carpentries.github.io"
carpentries_site: "https://carpentries.org/"
-dc_site: "http://datacarpentry.org"
+dc_site: "https://datacarpentry.org"
example_repo: "https://github.com/carpentries/lesson-example"
example_site: "https://carpentries.github.io/lesson-example"
lc_site: "https://librarycarpentry.org/"
@@ -50,13 +51,8 @@ workshop_site: "https://carpentries.github.io/workshop-template"
cc_by_human: "https://creativecommons.org/licenses/by/4.0/"
# Surveys.
-swc_pre_survey: "https://www.surveymonkey.com/r/swc_pre_workshop_v1?workshop_id="
-swc_post_survey: "https://www.surveymonkey.com/r/swc_post_workshop_v1?workshop_id="
-training_post_survey: "https://www.surveymonkey.com/r/post-instructor-training"
-dc_pre_survey: "https://www.surveymonkey.com/r/dcpreworkshopassessment?workshop_id="
-dc_post_survey: "https://www.surveymonkey.com/r/dcpostworkshopassessment?workshop_id="
-lc_pre_survey: "https://www.surveymonkey.com/r/lcpreworkshopsurvey?workshop_id="
-lc_post_survey: "https://www.surveymonkey.com/r/lcpostworkshopsurvey?workshop_id="
+pre_survey: "https://carpentries.typeform.com/to/wi32rS?slug="
+post_survey: "https://carpentries.typeform.com/to/UgVdRQ?slug="
instructor_pre_survey: "https://www.surveymonkey.com/r/instructor_training_pre_survey?workshop_id="
instructor_post_survey: "https://www.surveymonkey.com/r/instructor_training_post_survey?workshop_id="
@@ -96,6 +92,9 @@ exclude:
- Makefile
- bin/
- .Rproj.user/
+ - .vendor/
+ - vendor/
+ - .docker-vendor/
# Turn on built-in syntax highlighting.
highlighter: rouge
diff --git a/bin/boilerplate/_extras/figures.md b/bin/boilerplate/_extras/figures.md
index ee5b650..0012c88 100644
--- a/bin/boilerplate/_extras/figures.md
+++ b/bin/boilerplate/_extras/figures.md
@@ -3,11 +3,17 @@ title: Figures
---
{% include base_path.html %}
+{% include manual_episode_order.html %}
-{% comment %}
-Create anchor for each one of the episodes.
-{% endcomment %}
-{% for episode in site.episodes %}
+
+{% comment %} Create anchor for each one of the episodes. {% endcomment %}
+
+{% for lesson_episode in lesson_episodes %}
+ {% if site.episode_order %}
+ {% assign episode = site.episodes | where: "slug", lesson_episode | first %}
+ {% else %}
+ {% assign episode = lesson_episode %}
+ {% endif %}
{% endfor %}
diff --git a/bin/chunk-options.R b/bin/chunk-options.R
index 6bd4aef..8e0d62a 100644
--- a/bin/chunk-options.R
+++ b/bin/chunk-options.R
@@ -37,24 +37,34 @@ opts_chunk$set(tidy = FALSE, results = "markup", comment = NA,
# are properly formatted when the site is built.
hook_in <- function(x, options) {
+ lg <- tolower(options$engine)
+ style <- paste0(".language-", lg)
+
stringr::str_c("\n\n~~~\n",
- paste0(x, collapse="\n"),
- "\n~~~\n{: .language-r}\n\n")
+ paste0(x, collapse="\n"),
+ "\n~~~\n{: ", style, "}\n\n")
}
hook_out <- function(x, options) {
x <- gsub("\n$", "", x)
stringr::str_c("\n\n~~~\n",
- paste0(x, collapse="\n"),
- "\n~~~\n{: .output}\n\n")
+ paste0(x, collapse="\n"),
+ "\n~~~\n{: .output}\n\n")
}
hook_error <- function(x, options) {
x <- gsub("\n$", "", x)
stringr::str_c("\n\n~~~\n",
- paste0(x, collapse="\n"),
- "\n~~~\n{: .error}\n\n")
+ paste0(x, collapse="\n"),
+ "\n~~~\n{: .error}\n\n")
+}
+
+hook_warning <- function(x, options) {
+ x <- gsub("\n$", "", x)
+ stringr::str_c("\n\n~~~\n",
+ paste0(x, collapse = "\n"),
+ "\n~~~\n{: .warning}\n\n")
}
-knit_hooks$set(source = hook_in, output = hook_out, warning = hook_error,
- error = hook_error, message = hook_out)
+knit_hooks$set(source = hook_in, output = hook_out, warning = hook_warning,
+ error = hook_error, message = hook_out)
diff --git a/bin/dependencies.R b/bin/dependencies.R
new file mode 100644
index 0000000..676b050
--- /dev/null
+++ b/bin/dependencies.R
@@ -0,0 +1,55 @@
+install_required_packages <- function(lib = NULL, repos = getOption("repos")) {
+
+ if (is.null(lib)) {
+ lib <- .libPaths()
+ }
+
+ message("lib paths: ", paste(lib, collapse = ", "))
+ missing_pkgs <- setdiff(
+ c("rprojroot", "desc", "remotes", "renv"),
+ rownames(installed.packages(lib.loc = lib))
+ )
+
+ install.packages(missing_pkgs, lib = lib, repos = repos)
+
+}
+
+find_root <- function() {
+
+ cfg <- rprojroot::has_file_pattern("^_config.y*ml$")
+ root <- rprojroot::find_root(cfg)
+
+ root
+}
+
+identify_dependencies <- function() {
+
+ root <- find_root()
+
+ required_pkgs <- unique(c(
+ ## Packages for episodes
+ renv::dependencies(file.path(root, "_episodes_rmd"), progress = FALSE, error = "ignore")$Package,
+ ## Packages for tools
+ renv::dependencies(file.path(root, "bin"), progress = FALSE, error = "ignore")$Package
+ ))
+
+ required_pkgs
+}
+
+create_description <- function(required_pkgs) {
+ d <- desc::description$new("!new")
+ lapply(required_pkgs, function(x) d$set_dep(x))
+ d$write("DESCRIPTION")
+}
+
+install_dependencies <- function(required_pkgs, ...) {
+
+ create_description(required_pkgs)
+ on.exit(file.remove("DESCRIPTION"))
+ remotes::install_deps(dependencies = TRUE, ...)
+
+ if (require("knitr") && packageVersion("knitr") < '1.9.19') {
+ stop("knitr must be version 1.9.20 or higher")
+ }
+
+}
diff --git a/bin/generate_md_episodes.R b/bin/generate_md_episodes.R
index 7f37a7b..ce9e8aa 100644
--- a/bin/generate_md_episodes.R
+++ b/bin/generate_md_episodes.R
@@ -1,34 +1,7 @@
generate_md_episodes <- function() {
- library("methods")
-
- if (!require("remotes", quietly = TRUE)) {
- install.packages("remotes", repos = c(CRAN = "https://cloud.r-project.org/"))
- }
-
- if (!require("requirements", quietly = TRUE)) {
- remotes::install_github("hadley/requirements")
- }
-
- required_pkgs <- unique(c(
- ## Packages for episodes
- requirements:::req_dir("_episodes_rmd"),
- ## Pacakges for tools
- requirements:::req_dir("bin")
- ))
-
- missing_pkgs <- setdiff(required_pkgs, rownames(installed.packages()))
-
- if (length(missing_pkgs)) {
- message("Installing missing required packages: ",
- paste(missing_pkgs, collapse=", "))
- install.packages(missing_pkgs)
- }
-
- if (require("knitr") && packageVersion("knitr") < '1.9.19')
- stop("knitr must be version 1.9.20 or higher")
-
- ## get the Rmd file to process from the command line, and generate the path for their respective outputs
+ ## get the Rmd file to process from the command line, and generate the path
+ ## for their respective outputs
args <- commandArgs(trailingOnly = TRUE)
if (!identical(length(args), 2L)) {
stop("input and output file must be passed to the script")
@@ -40,20 +13,28 @@ generate_md_episodes <- function() {
## knit the Rmd into markdown
knitr::knit(src_rmd, output = dest_md)
- # Read the generated md files and add comments advising not to edit them
- vapply(dest_md, function(y) {
- con <- file(y)
- mdfile <- readLines(con)
- if (mdfile[1] != "---")
- stop("Input file does not have a valid header")
- mdfile <- append(mdfile, "# Please do not edit this file directly; it is auto generated.", after = 1)
- mdfile <- append(mdfile, paste("# Instead, please edit",
- basename(y), "in _episodes_rmd/"), after = 2)
- writeLines(mdfile, con)
- close(con)
- return(paste("Warning added to YAML header of", y))
- },
- character(1))
+ # Read the generated md files and add comments advising not to edit them
+ add_no_edit_comment <- function(y) {
+ con <- file(y)
+ mdfile <- readLines(con)
+ if (mdfile[1] != "---")
+ stop("Input file does not have a valid header")
+ mdfile <- append(
+ mdfile,
+ "# Please do not edit this file directly; it is auto generated.",
+ after = 1
+ )
+ mdfile <- append(
+ mdfile,
+ paste("# Instead, please edit", basename(y), "in _episodes_rmd/"),
+ after = 2
+ )
+ writeLines(mdfile, con)
+ close(con)
+ return(paste("Warning added to YAML header of", y))
+ }
+
+ vapply(dest_md, add_no_edit_comment, character(1))
}
generate_md_episodes()
diff --git a/bin/install_r_deps.sh b/bin/install_r_deps.sh
new file mode 100755
index 0000000..0280f24
--- /dev/null
+++ b/bin/install_r_deps.sh
@@ -0,0 +1 @@
+Rscript -e "source(file.path('bin', 'dependencies.R')); install_required_packages(); install_dependencies(identify_dependencies())"
diff --git a/bin/lesson_check.py b/bin/lesson_check.py
index 84c28f3..d14ace7 100755
--- a/bin/lesson_check.py
+++ b/bin/lesson_check.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
"""
Check lesson files and their contents.
"""
@@ -29,19 +27,19 @@
# specially. This list must include all the Markdown files listed in the
# 'bin/initialize' script.
REQUIRED_FILES = {
- '%/CODE_OF_CONDUCT.md': True,
- '%/CONTRIBUTING.md': False,
- '%/LICENSE.md': True,
- '%/README.md': False,
- '%/_extras/discuss.md': True,
- '%/_extras/guide.md': True,
- '%/index.md': True,
- '%/reference.md': True,
- '%/setup.md': True,
+ 'CODE_OF_CONDUCT.md': True,
+ 'CONTRIBUTING.md': False,
+ 'LICENSE.md': True,
+ 'README.md': False,
+ os.path.join('_extras', 'discuss.md'): True,
+ os.path.join('_extras', 'guide.md'): True,
+ 'index.md': True,
+ 'reference.md': True,
+ 'setup.md': True,
}
# Episode filename pattern.
-P_EPISODE_FILENAME = re.compile(r'/_episodes/(\d\d)-[-\w]+.md$')
+P_EPISODE_FILENAME = re.compile(r'(\d\d)-[-\w]+.md$')
# Pattern to match lines ending with whitespace.
P_TRAILING_WHITESPACE = re.compile(r'\s+$')
@@ -55,6 +53,9 @@
# Pattern to match reference links (to resolve internally-defined references).
P_INTERNAL_LINK_DEF = re.compile(r'^\[([^\]]+)\]:\s*(.+)')
+# Pattern to match {% include ... %} statements
+P_INTERNAL_INCLUDE_LINK = re.compile(r'^{% include ([^ ]*) %}$')
+
# What kinds of blockquotes are allowed?
KNOWN_BLOCKQUOTES = {
'callout',
@@ -76,6 +77,9 @@
'source',
'language-bash',
'html',
+ 'language-c',
+ 'language-cmake',
+ 'language-cpp',
'language-make',
'language-matlab',
'language-python',
@@ -176,7 +180,7 @@ def check_config(reporter, source_dir):
reporter.check_field(config_file, 'configuration',
config, 'kind', 'lesson')
reporter.check_field(config_file, 'configuration',
- config, 'carpentry', ('swc', 'dc', 'lc', 'cp'))
+ config, 'carpentry', ('swc', 'dc', 'lc', 'cp','incubator'))
reporter.check_field(config_file, 'configuration', config, 'title')
reporter.check_field(config_file, 'configuration', config, 'email')
@@ -208,29 +212,44 @@ def read_references(reporter, ref_path):
{symbolic_name : URL}
"""
+ if not ref_path:
+ raise Warning("No filename has been provided.")
+
result = {}
urls_seen = set()
- if ref_path:
- with open(ref_path, 'r') as reader:
- for (num, line) in enumerate(reader):
- line_num = num + 1
- m = P_INTERNAL_LINK_DEF.search(line)
- require(m,
- '{0}:{1} not valid reference:\n{2}'.format(ref_path, line_num, line.rstrip()))
- name = m.group(1)
- url = m.group(2)
- require(name,
- 'Empty reference at {0}:{1}'.format(ref_path, line_num))
- reporter.check(name not in result,
- ref_path,
- 'Duplicate reference {0} at line {1}',
- name, line_num)
- reporter.check(url not in urls_seen,
- ref_path,
- 'Duplicate definition of URL {0} at line {1}',
- url, line_num)
- result[name] = url
- urls_seen.add(url)
+
+ with open(ref_path, 'r', encoding='utf-8') as reader:
+ for (num, line) in enumerate(reader, 1):
+
+ if P_INTERNAL_INCLUDE_LINK.search(line): continue
+
+ m = P_INTERNAL_LINK_DEF.search(line)
+
+ message = '{}: {} not a valid reference: {}'
+ require(m, message.format(ref_path, num, line.rstrip()))
+
+ name = m.group(1)
+ url = m.group(2)
+
+ message = 'Empty reference at {0}:{1}'
+ require(name, message.format(ref_path, num))
+
+ unique_name = name not in result
+ unique_url = url not in urls_seen
+
+ reporter.check(unique_name,
+ ref_path,
+ 'Duplicate reference name {0} at line {1}',
+ name, num)
+
+ reporter.check(unique_url,
+ ref_path,
+ 'Duplicate definition of URL {0} at line {1}',
+ url, num)
+
+ result[name] = url
+ urls_seen.add(url)
+
return result
@@ -254,7 +273,7 @@ def check_fileset(source_dir, reporter, filenames_present):
"""Are all required files present? Are extraneous files present?"""
# Check files with predictable names.
- required = [p.replace('%', source_dir) for p in REQUIRED_FILES]
+ required = [os.path.join(source_dir, p) for p in REQUIRED_FILES]
missing = set(required) - set(filenames_present)
for m in missing:
reporter.add(None, 'Missing required file {0}', m)
@@ -264,7 +283,10 @@ def check_fileset(source_dir, reporter, filenames_present):
for filename in filenames_present:
if '_episodes' not in filename:
continue
- m = P_EPISODE_FILENAME.search(filename)
+
+ # split path to check episode name
+ base_name = os.path.basename(filename)
+ m = P_EPISODE_FILENAME.search(base_name)
if m and m.group(1):
seen.append(m.group(1))
else:
@@ -342,7 +364,7 @@ def check_line_lengths(self):
n > MAX_LINE_LEN) and (not l.startswith('!'))]
self.reporter.check(not over,
self.filename,
- 'Line(s) are too long: {0}',
+ 'Line(s) too long: {0}',
', '.join([str(i) for i in over]))
def check_trailing_whitespace(self):
@@ -538,8 +560,7 @@ def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
(re.compile(r'README\.md'), CheckNonJekyll),
(re.compile(r'index\.md'), CheckIndex),
(re.compile(r'reference\.md'), CheckReference),
- (re.compile(r'_episodes/.*\.md'), CheckEpisode),
- (re.compile(r'aio\.md'), CheckNonJekyll),
+ (re.compile(os.path.join('_episodes', '*\.md')), CheckEpisode),
(re.compile(r'.*\.md'), CheckGeneric)
]
diff --git a/bin/lesson_initialize.py b/bin/lesson_initialize.py
index a5eb6d0..2f7b8e6 100755
--- a/bin/lesson_initialize.py
+++ b/bin/lesson_initialize.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
"""Initialize a newly-created repository."""
@@ -14,12 +12,11 @@
'CONTRIBUTING.md',
'README.md',
'_config.yml',
- '_episodes/01-introduction.md',
- '_extras/about.md',
- '_extras/discuss.md',
- '_extras/figures.md',
- '_extras/guide.md',
- 'aio.md',
+ os.path.join('_episodes', '01-introduction.md'),
+ os.path.join('_extras', 'about.md'),
+ os.path.join('_extras', 'discuss.md'),
+ os.path.join('_extras', 'figures.md'),
+ os.path.join('_extras', 'guide.md'),
'index.md',
'reference.md',
'setup.md',
@@ -42,7 +39,7 @@ def main():
# Create.
for path in BOILERPLATE:
shutil.copyfile(
- "bin/boilerplate/{}".format(path),
+ os.path.join('bin', 'boilerplate', path),
path
)
diff --git a/bin/repo_check.py b/bin/repo_check.py
index af4b782..9bf5c59 100755
--- a/bin/repo_check.py
+++ b/bin/repo_check.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
"""
Check repository settings.
"""
@@ -22,7 +20,7 @@
# Pattern to match Git command-line output for remotes => (user name, project name).
-P_GIT_REMOTE = re.compile(r'upstream\s+[^:]+:([^/]+)/([^.]+)\.git\s+\(fetch\)')
+P_GIT_REMOTE = re.compile(r'upstream\s+(?:https://|git@)github.com[:/]([^/]+)/([^.]+)(\.git)?\s+\(fetch\)')
# Repository URL format string.
F_REPO_URL = 'https://github.com/{0}/{1}/'
@@ -104,7 +102,7 @@ def get_repo_url(repo_url):
# Guess.
cmd = 'git remote -v'
p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE,
- close_fds=True, universal_newlines=True)
+ close_fds=True, universal_newlines=True, encoding='utf-8')
stdout_data, stderr_data = p.communicate()
stdout_data = stdout_data.split('\n')
matches = [P_GIT_REMOTE.match(line) for line in stdout_data]
diff --git a/bin/run-make-docker-serve.sh b/bin/run-make-docker-serve.sh
new file mode 100755
index 0000000..1e09178
--- /dev/null
+++ b/bin/run-make-docker-serve.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+bundle install
+bundle update
+exec bundle exec jekyll serve --host 0.0.0.0
diff --git a/bin/test_lesson_check.py b/bin/test_lesson_check.py
index 960059e..0981720 100755
--- a/bin/test_lesson_check.py
+++ b/bin/test_lesson_check.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
import unittest
import lesson_check
@@ -12,10 +10,8 @@ def setUp(self):
def test_file_list_has_expected_entries(self):
# For first pass, simply assume that all required files are present
- all_filenames = [filename.replace('%', '')
- for filename in lesson_check.REQUIRED_FILES]
- lesson_check.check_fileset('', self.reporter, all_filenames)
+ lesson_check.check_fileset('', self.reporter, lesson_check.REQUIRED_FILES)
self.assertEqual(len(self.reporter.messages), 0)
diff --git a/bin/util.py b/bin/util.py
index 522f5df..0e16d86 100644
--- a/bin/util.py
+++ b/bin/util.py
@@ -105,7 +105,7 @@ def read_markdown(parser, path):
"""
# Split and extract YAML (if present).
- with open(path, 'r') as reader:
+ with open(path, 'r', encoding='utf-8') as reader:
body = reader.read()
metadata_raw, metadata_yaml, body = split_metadata(path, body)
@@ -117,7 +117,7 @@ def read_markdown(parser, path):
# Parse Markdown.
cmd = 'ruby {0}'.format(parser)
p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE,
- close_fds=True, universal_newlines=True)
+ close_fds=True, universal_newlines=True, encoding='utf-8')
stdout_data, stderr_data = p.communicate(body)
doc = json.loads(stdout_data)
@@ -144,7 +144,7 @@ def split_metadata(path, text):
metadata_raw = pieces[1]
text = pieces[2]
try:
- metadata_yaml = yaml.load(metadata_raw)
+ metadata_yaml = yaml.load(metadata_raw, Loader=yaml.SafeLoader)
except yaml.YAMLError as e:
print('Unable to parse YAML header in {0}:\n{1}'.format(
path, e), file=sys.stderr)
@@ -160,8 +160,8 @@ def load_yaml(filename):
"""
try:
- with open(filename, 'r') as reader:
- return yaml.load(reader)
+ with open(filename, 'r', encoding='utf-8') as reader:
+ return yaml.load(reader, Loader=yaml.SafeLoader)
except (yaml.YAMLError, IOError) as e:
print('Unable to load YAML file {0}:\n{1}'.format(
filename, e), file=sys.stderr)
diff --git a/bin/workshop_check.py b/bin/workshop_check.py
index 0523d0c..15d954a 100755
--- a/bin/workshop_check.py
+++ b/bin/workshop_check.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
'''Check that a workshop's index.html metadata is valid. See the
docstrings on the checking functions for a summary of the checks.
'''
@@ -410,7 +408,7 @@ def main():
reporter = Reporter()
check_config(reporter, config_file)
check_unwanted_files(root_dir, reporter)
- with open(index_file) as reader:
+ with open(index_file, encoding='utf-8') as reader:
data = reader.read()
check_file(reporter, index_file, data)
reporter.report()
diff --git a/reference.md b/reference.md
index 8c82616..0165baf 100644
--- a/reference.md
+++ b/reference.md
@@ -4,6 +4,6 @@ layout: reference
## Glossary
-FIXME
+
{% include links.md %}
diff --git a/setup.md b/setup.md
index 4990943..f7e9c09 100644
--- a/setup.md
+++ b/setup.md
@@ -38,6 +38,6 @@ wget https://scw-aberystwyth.github.io/machine-learning-novice/data/worldbank-gd
wget https://scw-aberystwyth.github.io/machine-learning-novice/data/worldbank-gdp-outliers.csv
wget https://scw-aberystwyth.github.io/machine-learning-novice/data/gapminder-life-expectancy.csv
~~~
-{: .bash}
+{: .language-bash}
{% include links.md %}