diff --git a/.travis.yml b/.travis.yml
index c15462c68..7de79519d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,6 +6,7 @@ script:
 - "sh ./_test/ensure-stubs-compile.sh"
 - "sh ./_test/count-ignores.sh"
 - "./bin/fetch-configlet"
+- "sh ./_test/ensure-readmes-are-updated.sh"
 - "./bin/configlet lint ."
 sudo: false
 rust:
diff --git a/_test/ensure-readmes-are-updated.sh b/_test/ensure-readmes-are-updated.sh
new file mode 100644
index 000000000..5b09bc741
--- /dev/null
+++ b/_test/ensure-readmes-are-updated.sh
@@ -0,0 +1,36 @@
+if [ ! -x bin/configlet ]; then
+   echo "Improper configuration; configlet should exist in bin/ when this script is run"
+   echo "Ping a Rust track maintainer to fix this"
+   exit 1
+fi
+
+newline=$'\n  '
+
+missing_readmes=""
+wrong_readmes=""
+for exercise in $(git diff --name-only master..$(git rev-parse --abbrev-ref HEAD) | grep exercises/ | cut -d'/' -f2 -s | sort -fu); do
+   echo "Checking readme for $exercise"
+   readme_path="exercises/${exercise}/README.md"
+   if [ ! -f $readme_path ]; then
+      missing_readmes="$missing_readmes$newline$exercise"
+   else
+      existing_readme_checksum=$(md5sum $readme_path | cut -d' ' -f1)
+      # generate the new README
+      bin/configlet generate . --only "$exercise"
+      generated_readme_checksum=$(md5sum $readme_path | cut -d' ' -f1)
+
+      if [ $existing_readme_checksum != $generated_readme_checksum ]; then
+         wrong_readmes="$wrong_readmes$newline$exercise"
+      fi
+   fi
+done
+
+if [ -n "$missing_readmes" ]; then
+  echo "Exercises missing README.md:$missing_readmes"
+fi
+if [ -n "$wrong_readmes" ]; then
+  echo "Exercises with out-of-date README.md:$wrong_readmes"
+fi
+if [ -n "$missing_readmes" -o -n "$wrong_readmes" ]; then
+   exit 1
+fi