Added support for referencing remote schemas

README.md

@@ -108,6 +108,23 @@ As developer we're always looking for ways to improve and optimize our workflow,
 
 For a list of open source Swagger based libraries in many languages check here: http://swagger.io/open-source-integrations/
 
+## Development
+
+### Testing
+
+Testing is powered by [Mocha](https://mochajs.org/)/[Chai](http://chaijs.com/), and automated testing is run via [CircleCI](https://circleci.com/).
+
+At this stage, unit tests have not been written for all parts of the codebase.  However, new code should be tested, and unit tests for the existing code will be added in the future.
+
+Run `npm test` on the repository to start the automated tests.
+Some parts of testing can be configured using environment variables.
+
+- `OFFLINE=true`
+  Some tests use HTTP connections to test giving Spectacle remote API specifications.
+  Use `OFFLINE=true` to skip tests that require an internet connection.
+
+Include environment variables before calling `npm test`.  For example, `OFFLINE` mode can be enabled via `OFFLINE=true npm test`.
+
 ## More Information
 
 More info is available on the [Spectacle homepage](http://sourcey.com/spectacle).

app/lib/errors.js

@@ -0,0 +1,20 @@
+exports.LocalRefError = LocalRefError = function LocalRefError(message) {
+  this.name = "LocalRefError";
+  this.message = message || "Local Reference Error";
+  this.stack = (new Error()).stack;
+}
+
+LocalRefError.prototype = Object.create(Error.prototype);
+LocalRefError.prototype.constructor = LocalRefError;
+
+/**
+ * Used when the spec tree is improperly walked, and gets into an unrecoverable spot.
+*/
+exports.TreeWalkError = TreeWalkError = function TreeWalkError(message) {
+  this.name = "TreeWalkError";
+  this.message = message || "Error Walking Tree";
+  this.stack = (new Error()).stack;
+}
+
+TreeWalkError.prototype = Object.create(Error.prototype);
+TreeWalkError.prototype.constructor = TreeWalkError;

app/lib/json-reference.js

@@ -0,0 +1,60 @@
+var path = require("path");
+
+/**
+ * Applies a single JSON reference lookup to an object.  Does not resolve references in the returned document.
+ * @param {string} ref the JSON reference to search for - e.g. `"#/foo/bar"`
+ * @param {object} obj the object to find the referenced field in - e.g. `{"foo": {"bar": 5}}`
+ * @throws {ReferenceError} if the reference can't be followed.
+ * @return {*} The referenced element in the given object.
+*/
+function jsonSearch(ref, obj) {
+  var current = obj;
+  refs = ref.replace(/^#?\/?/, "").split("/").forEach(function(section) {
+    if(section.trim().length < 1) {
+      return;
+    }
+    if(current[section]) {
+      current = current[section];
+    }
+    else {
+      throw new ReferenceError("Couldn't evaluate JSON reference '"+ref+"': Couldn't find key "+section);
+    }
+  });
+  return current;
+}
+
+/**
+ * Resolve all local JSON references in a single document.  Resolves absolute references (`#/test/`) and relative paths
+ * (`../test`), but does not change any references to remote files.
+ * Mutates the given object with resolved references.
+ * @param {Object} doc the root JSON document that references are being resolved in.
+ * @param {Object} obj a section of the JSON document that is being evaluated.
+ * @param {String} ref the path to the current object inside the JSON document, as a JSON reference.
+*/
+function resolveLocal(doc, obj, ref) {
+  if(typeof obj !== "object") {
+    throw new TypeError("resolveLocal() must be given an object.  Given "+typeof obj+" ("+obj+")");
+  }
+  for(var k in obj) {
+    var val = obj[k];
+    if(typeof val !== "object") { continue; }
+    if(val.$ref) {
+      var $ref = val.$ref;
+      if($ref.indexOf("./") === 0 || $ref.indexOf("../") === 0) {
+        $ref = path.join(ref, k, $ref);
+      }
+      if($ref.indexOf("#/") === 0) {
+        Object.assign(val, jsonSearch($ref, doc));
+        delete val.$ref;
+      }
+    }
+    else {
+      resolveLocal(doc, val, path.join(ref, k));
+    }
+  }
+}
+
+module.exports = {
+  jsonSearch: jsonSearch,
+  resolveLocal: resolveLocal,
+};

app/lib/preprocessor.js

@@ -1,12 +1,18 @@
 var path = require('path'),
   _ = require('lodash');
 
-var httpMethods = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch'];
+var httpMethods = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', '$ref'];
 
 // Preprocessor for the Swagger JSON so that some of the logic can be taken
 // out of the template.
 
 module.exports = function(options, specData) {
+  if(!options.specFile) {
+    console.warn("[WARNING] preprocessor must be given 'options.specFile'.  Defaulting to 'cwd'.");
+    options.specFile = process.cwd();
+  }
+  specData["x-spec-path"] = options.specFile;
+
   var copy = _.cloneDeep(specData);
   var tagsByName = _.keyBy(copy.tags, 'name');
 
@@ -65,5 +71,11 @@ module.exports = function(options, specData) {
     // If there are multiple tags, we show the tag-based summary
     copy.showTagSummary = copy.tags.length > 1
   }
+
+  var replaceRefs = require("./resolve-references").replaceRefs;
+  replaceRefs(path.dirname(copy["x-spec-path"]), copy, copy, "");
+
   return copy;
 }
+
+module.exports.httpMethods = httpMethods;

app/lib/reference-contexts.js

@@ -0,0 +1,52 @@
+/**
+ * Methods that can recognize the current "context" of a `$ref` reference.
+ * This allows sections to be included differently; for instance a reference for `/responses/500/schema` could be
+ * inserted into the global definitions, instead of being included multiple times in the document.
+*/
+
+/**
+ * Recognizes 'schema' elements in 'responses' to be definitions.
+ * @param {String} ref the JSON reference
+ * @return {Boolean} `true` if the current path is to a response schema.
+*/
+function responseSchemaDefinition(ref) {
+  var parts = ref.split("/");
+  return parts.length > 4 && parts.indexOf("schema") === parts.length - 2 &&
+    parts.indexOf("responses") === parts.length - 4;
+}
+
+/**
+ * Recognize items in property arrays.
+ * @param {String} ref the JSON reference.
+ * @return {Boolean} `true` if the current path is to an array inside a property.
+*/
+function propertyArrayDefinition(ref) {
+  var parts = ref.split("/");
+  return parts.length > 2 && parts.lastIndexOf("items") === parts.length - 2;
+}
+
+/**
+ * Recognizes different types of definitions that could be moved to the global 'definitions'.
+ * @param {String} ref the JSON reference
+ * @return {Boolean} `true` if the current path is an OpenAPI `definition`.
+*/
+function definition(ref) {
+  return responseSchemaDefinition(ref) || propertyArrayDefinition(ref);
+}
+
+/**
+ * Recognize URL paths.
+ * @param {String} ref the JSON reference.
+ * @return {Boolean} `true` if the current path is an OpenAPI `path`.
+*/
+function path(ref) {
+  var parts = ref.split("/");
+  return parts.length === 3 && parts.lastIndexOf("paths") === parts.length - 3 && parts[1].length > 0;
+}
+
+module.exports = {
+  responseSchemaDefinition: responseSchemaDefinition,
+  propertyArrayDefinition: propertyArrayDefinition,
+  definition: definition,
+  path: path,
+};

app/lib/resolve-references.js

@@ -0,0 +1,200 @@
+var fs = require("fs");
+var path = require("path");
+var yaml = require("js-yaml");
+var request = require("request-sync");
+var _ = require("lodash");
+var pathUtils = require("./urls");
+var contexts = require("./reference-contexts");
+var resolveLocal = require("./json-reference").resolveLocal;
+var jsonSearch = require("./json-reference").jsonSearch;
+var LocalRefError = require("./errors").LocalRefError;
+var TreeWalkError = require("./errors").TreeWalkError;
+
+/**
+ * Utilities for the preprocessor that can resolve external references.
+*/
+
+/**
+ * Stores a parsed copy of referenced files to improve preformance.
+*/
+var _cache = {};
+
+/**
+ * The list of avalible HTTP methods.
+*/
+var httpMethods = require("./preprocessor").httpMethods;
+
+/**
+ * Determines if a reference is relative to the current file.
+ * @param {string} ref The file path or URL referenced.
+ * @return {boolean} `true` if the reference points to the current file.
+*/
+function localReference(ref) {
+  return (typeof ref.trim === "function" && ref.trim().indexOf("#") === 0) ||
+         (typeof ref.indexOf === "function" && ref.indexOf("#") === 0);
+}
+
+/**
+ * Fetches a section of an external file.
+ * @param {string} ref The file path or URL referenced.
+ * @return {object} The section requested.
+ * @throws {LocalRefError} if 'ref' points to a local definition (starts with `#`)
+ * @todo Improve YAML detection
+ * @todo Improve cache preformance by ensuring paths are normalized, etc.
+ * @todo Test failure
+*/
+function fetchReference(ref) {
+
+  if(localReference(ref)) {
+    throw new LocalRefError("fetchReference('"+ref+"') given a reference to the current file.");
+  }
+
+  var file = ref.split("#", 1)[0];
+  var refPath = ref.substr(file.length + 1);
+
+  var src = null;
+
+  if(_cache[file]) {
+    src = _cache[file];
+  }
+  else {
+    if(pathUtils.absoluteURL(file)) {
+      src = request(file).body;
+    }
+    else {
+      src = fs.readFileSync(file, "utf8");
+    }
+    if(file.indexOf(".yml") > -1 || file.indexOf(".yaml") > -1) {
+      src = yaml.safeLoad(src);
+    }
+    else {
+      src = JSON.parse(src);
+    }
+    _cache[file] = src;
+  }
+
+  if(refPath.length > 0) {
+    src = jsonSearch(refPath, src);
+  }
+
+  if(src.$ref && typeof src.$ref === "string" && !localReference(src.$ref)) {
+    src = fetchReference(pathUtils.join(path.dirname(ref), src.$ref));
+  }
+
+  return src;
+}
+
+/**
+ * Replace external references in a specification with the contents.  Mutates the given objects.
+ * Mutually recursive with `replaceRefs`.
+ * @param {string} cwd The path to the file containing a reference.
+ * @param {object} top The top-level specification file.
+ * @param {object} obj The object referencing an external file.
+ * @param {string} context The current reference path, e.g. `"#/paths/%2F/"`
+ * @todo test failure
+*/
+function replaceReference(cwd, top, obj, context) {
+  var ref = pathUtils.join(cwd, obj.$ref);
+  var external = pathUtils.relative(path.dirname(top["x-spec-path"]), ref);
+  var referenced = module.exports.fetchReference(ref);
+  if(typeof referenced === "object") {
+    resolveLocal(referenced, referenced, "#/");
+    referenced["x-external"] = external;
+    module.exports.replaceRefs(path.dirname(ref), top, referenced, context);
+  }
+  if(contexts.definition(context)) {
+    if(!top.definitions) { top.definitions = {}; }
+    if(!top.definitions[external]) { top.definitions[external] = referenced; }
+    Object.assign(obj, { "$ref": "#/definitions/"+external.replace("/", "%2F") });
+  }
+  else if(contexts.path(context)) {
+    Object.keys(referenced).forEach(function(method) {
+      if(httpMethods.indexOf(method) < 0) {
+        delete path[method];
+        return;
+      }
+      var operation = referenced[method];
+      operation.method = method;
+      var operationTags = operation.tags || ["default"];
+      operationTags.forEach(function(tag) {
+        top.tags = top.tags || [];
+        var tagDef = _.find(top.tags, {name: tag});
+        if(!tagDef) {
+          tagDef = {name: tag, operations: []};
+          top.tags.push(tagDef);
+        }
+        tagDef.operations = tagDef.operations || [];
+        tagDef.operations.push(operation);
+      });
+    });
+    Object.assign(obj, referenced);
+    delete obj.$ref;
+  }
+  else {
+    Object.assign(obj, referenced);
+    delete obj.$ref;
+  }
+}
+
+/**
+ * Walk an given Swagger tree, and replace all JSON references in the form of `{"$ref": "<path>"}` with the proper
+ * contents.
+ * Recursive, as well as being mutually recursive with `replaceReference`.
+ * @param {string} cwd The path of the current file
+ * @param {object} top The top-level specification file.
+ * @param {object} obj The Swagger tree to evaluate.
+ * @param {string} context The current reference path, e.g. `"#/paths/%2F/"`
+ * @throws {TreeWalkError} if `obj` itself is a reference.
+ * @todo Test failure
+ * @todo Test edge cases (remote relative ref, etc.)
+*/
+function replaceRefs(cwd, top, obj, context) {
+
+  if(typeof cwd !== "string" || cwd.length < 1) {
+    throw new Error("replaceRefs must be given a 'cwd'.  Given '"+cwd+"'");
+  }
+  if(typeof obj !== "object") {
+    console.warn("[WARN] replaceRefs() must be given an object for 'obj'.  Given "+typeof obj+" ("+obj+")");
+    return; }
+
+  if(obj.$ref) {
+    throw new TreeWalkError("Walked too deep in the tree looking for references.  Can't resolve reference " +
+      obj.$ref + " in "+cwd+".");
+  }
+
+  for(var k in obj) {
+    var val = obj[k];
+    if(typeof val !== "object") { continue; }
+
+    if(val.$ref) {
+
+      if(localReference(val.$ref)) {
+        if((cwd === top["x-spec-path"]) || (cwd === path.dirname(top["x-spec-path"]))) { continue; }
+        throw new Error(
+          "Can't deal with internal references in external files yet.  Got: '"+val.$ref+"'.");
+      }
+
+      try {
+        module.exports.replaceReference(cwd, top, val, context + k + "/");
+      }
+      catch (e) {
+        console.error("replaceRefs(): Couldn't replace reference '"+val.$ref+"' from '"+
+          cwd+"'.  Reference path: #/"+context);
+        throw e;
+      }
+
+      continue;
+    }
+
+    replaceRefs(cwd, top, val, context + k + "/");
+  }
+
+}
+
+module.exports = {
+  _cache: _cache,
+  localReference: localReference,
+  fetchReference: fetchReference,
+  replaceReference: replaceReference,
+  replaceRefs: replaceRefs,
+}