Added option to prevent pages from appearing in the robots.txt at all.

.babelrc

@@ -0,0 +1,8 @@
+{
+  "presets": [
+    "es2015"
+  ],
+  "plugins": [
+    "add-module-exports"
+  ]
+}

lib/index.js

@@ -1,90 +1,90 @@
 "use strict";
 
-var _prototypeProperties = function (child, staticProps, instanceProps) { if (staticProps) Object.defineProperties(child, staticProps); if (instanceProps) Object.defineProperties(child.prototype, instanceProps); };
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
 
-var _inherits = function (subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) subClass.__proto__ = superClass; };
+var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();
 
-var _classCallCheck = function (instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } };
-
-module.exports = function (env, callback) {
+exports.default = function (env, callback) {
 
   // uses 'wintersmith-contents' as implemented in 'wintersmith-sitemap':
   // https://github.com/xavierdutreilh/wintersmith-sitemap
-  var getPages = function (contents) {
+  var getPages = function getPages(contents) {
     return env.helpers.contents.list(contents).filter(function (item) {
       return item instanceof env.plugins.MarkdownPage;
     });
   };
 
   var robotsView = function robotsView(env, locals, contents, templates, callback) {
-    var url = locals.url;
-    var sitemap = locals.sitemap;
-    var noindex = locals.noindex;
-    var ua = "User-agent: *";
+    var url = locals.url,
+        sitemap = locals.sitemap,
+        noindex = locals.noindex,
+        ua = "User-agent: *";
 
     // Global block, skip the rest
-    if (noindex) {
-      return callback(null, new Buffer(ua + "\nDisallow: /"));
-    }var disallowedPages = getPages(contents).filter(function (page) {
-      return page.metadata.noindex;
+
+    if (noindex) return callback(null, new Buffer(ua + "\nDisallow: /"));
+
+    var disallowedPages = getPages(contents).filter(function (page) {
+      return page.metadata.noindex && page.metadata.noindex !== "exclude";
     }).map(function (page) {
       return "Disallow: " + page.url;
     });
 
     var sitemapURL = sitemap ? "Sitemap: " + url + "/" + sitemap : null;
 
-    var robots = [].concat(ua, disallowedPages, sitemapURL).join("\n");
+    var robots = [].concat(ua, disallowedPages, sitemapURL).join('\n');
 
     callback(null, new Buffer(robots));
   };
 
-  var Robots = (function (_env$plugins$Page) {
+  var Robots = function (_env$plugins$Page) {
+    _inherits(Robots, _env$plugins$Page);
+
     function Robots() {
       _classCallCheck(this, Robots);
 
-      if (_env$plugins$Page != null) {
-        _env$plugins$Page.apply(this, arguments);
-      }
+      return _possibleConstructorReturn(this, (Robots.__proto__ || Object.getPrototypeOf(Robots)).apply(this, arguments));
     }
 
-    _inherits(Robots, _env$plugins$Page);
-
-    _prototypeProperties(Robots, null, {
-      filename: {
-        get: function () {
-          return "robots.txt";
-        },
-        configurable: true
-      },
-      getFilename: {
-        value: function getFilename() {
-          return this.filename;
-        },
-        writable: true,
-        configurable: true
-      },
-      view: {
-        get: function () {
-          return robotsView;
-        },
-        configurable: true
-      },
-      getView: {
-        value: function getView() {
-          return this.view;
-        },
-        writable: true,
-        configurable: true
+    _createClass(Robots, [{
+      key: "getFilename",
+      value: function getFilename() {
+        return this.filename;
       }
-    });
+    }, {
+      key: "getView",
+      value: function getView() {
+        return this.view;
+      }
+    }, {
+      key: "filename",
+      get: function get() {
+        return 'robots.txt';
+      }
+    }, {
+      key: "view",
+      get: function get() {
+        return robotsView;
+      }
+    }]);
 
     return Robots;
-  })(env.plugins.Page);
+  }(env.plugins.Page);
 
-  env.registerGenerator("robots", function (contents, callback) {
-    var tree = { "robots.txt": new Robots() };
+  env.registerGenerator('robots', function (contents, callback) {
+    var tree = { 'robots.txt': new Robots() };
     callback(null, tree);
   });
 
   callback();
-};
+};
+
+function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
+
+function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; }
+
+function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; }
+
+module.exports = exports["default"];

package.json

@@ -19,11 +19,15 @@
   "main": "lib/",
   "scripts": {
     "prepublish": "babel -d ./lib ./src",
-    "test": "mocha --compilers babel:babel/register"
+    "test": "mocha --compilers js:babel-register"
   },
   "devDependencies": {
-    "babel": "^4.x",
-    "mocha": "^2.1.0"
+    "babel": "^6.5.2",
+    "babel-cli": "^6.18.0",
+    "babel-plugin-add-module-exports": "^0.2.1",
+    "babel-preset-es2015": "^6.18.0",
+    "babel-register": "^6.18.0",
+    "mocha": "^2.5.3"
   },
   "dependencies": {
     "wintersmith": "^2.1.0",

readme.md

@@ -41,3 +41,11 @@ Set per-page options at the top of your Markdown files. For instance, you can pr
 noindex: true
 ---
 ```
+
+The above will generate a `Disallow: /path/to/page` entry in your `robots.txt`. which might not always be desirable (e.g., if a plugin transforms some page into some different form entirely).  You can completely ignore a page with the following:
+
+```yaml
+---
+noindex: exclude
+---
+```

src/index.js

@@ -1,5 +1,5 @@
 export default function (env, callback) {
-  
+
   // uses 'wintersmith-contents' as implemented in 'wintersmith-sitemap':
   // https://github.com/xavierdutreilh/wintersmith-sitemap
   var getPages = (contents) => {
@@ -13,14 +13,15 @@ export default function (env, callback) {
 
     // Global block, skip the rest
     if (noindex) return callback(null, new Buffer(ua + "\nDisallow: /"));
-    
+
     var disallowedPages =
         getPages(contents)
-        .filter( page => page.metadata.noindex )
-        .map( page => `Disallow: ${page.url}`);
+        .filter(page => page.metadata.noindex &&
+                        page.metadata.noindex !== "exclude")
+        .map(page => `Disallow: ${page.url}`);
 
     var sitemapURL = sitemap ? (`Sitemap: ${url}/${sitemap}`) : null;
-    
+
     var robots = []
         .concat(ua, disallowedPages, sitemapURL)
         .join('\n');
@@ -29,7 +30,7 @@ export default function (env, callback) {
   };
 
   class Robots extends env.plugins.Page {
-    
+
     get filename() {
       return 'robots.txt';
     }
@@ -47,5 +48,5 @@ export default function (env, callback) {
     callback(null, tree);
   });
 
-  callback();  
+  callback();
 }