diff --git a/README.md b/README.md index f5f0a615d3..5767f7273b 100644 --- a/README.md +++ b/README.md @@ -36,21 +36,76 @@ There are three metadata files: ## Test Data Format (canonical-data.json) This data can be incorporated into test programs manually or extracted by a -program. The file contains a single JSON object with a key for documentation -and keys for various tests that may be meaningful for a problem. - -The documentation uses the key "#" with a list of strings as the value. -These strings document how the problem readme (`description.md`) is generally -interpreted in test programs across different languages. In addition to a -mainstream implementation path, this information can also document significant -variations. - -Each test case has the the following keys: -- description: which will be used to name each generated test - - The description should not simply explain **what** each case is (that is redundant information) - - The description should explain **why** each case is there. For example, what kinds of implementation mistakes might this case help us find? -- 'variable names': one or more variable names with values which will be passed to the solution method -- expected: the expected result +program. The file format is described in `canonical-schema.json`, but it +is easier to understand with a example: + +```json +{ "exercise": "foobar" +, "version" : "1.0.0" +, "comments": + [ " Comments are always optional and can be used almost anywhere. " + , " " + , " They usually document how the exercise's readme ('description.md') " + , " is generally interpreted in test programs across different " + , " languages. " + , " " + , " In addition to a mainstream implementation path, this information " + , " can also document significant variations. " + ] +, "cases": + [ { "comments": + [ " A test case must have a 'description' and a 'property'. " + , " Anything else is optional. " + , " " + , " The 'property' is a string in lowerCamelCase identifying " + , " the type of test, but most of the times it is just the " + , " name of a function being tested. " + , " " + , " Test cases can have any number of additional keys, and " + , " most of them also have an 'expected' one, defining the " + , " value a test should return. " + ] + , "description": "Foo'ing a word returns it reversed" + , "property" : "foo" + , "input" : "lion" + , "expected" : "noil" + } + , { "description": "Bar'ing a name returns its parts combined" + , "property" : "bar" + , "firstName" : "Alan" + , "lastName" : "Smithee" + , "expected" : "ASlmainthee" + } + , { "comments": + [ " Test cases can be arbitrarily grouped with a description " + , " to make organization easier. " + ] + , "description": "Abnormal inputs: numbers" + , "cases": + [ { "description": "Foo'ing a number returns nothing" + , "property" : "foo" + , "input" : "42" + , "expected" : null + } + , { "description": "Bar'ing a name with numbers gives an error" + , "property" : "bar" + , "firstName" : "HAL" + , "lastName" : "9000" + , "expected" : { "error": "You should never bar a number" } + } + ] + } + ] +} + +``` + +Keep in mind that the description should not simply explain **what** each case +is (that is redundant information) but also **why** each case is there. For +example, what kinds of implementation mistakes might this case help us find? + +There are also some convention about `expected` that you must follow: + - if the input is valid but there is no result for the input, the value at `"expected"` should be `null`. - if an error is expected (because the input is invalid, or any other reason), the value at `"expected"` should be an object containing exactly one property, `"error"`, whose value is a string. - The string should explain why the error would occur. diff --git a/canonical-schema.json b/canonical-schema.json new file mode 100644 index 0000000000..1c7ce5fada --- /dev/null +++ b/canonical-schema.json @@ -0,0 +1,137 @@ +{ + "comments": + [ " This is a JSON Schema for 'canonical-data.json' files. " + , " " + , " It enforces just a general structure for all exercises, " + , " without specifying how the test data should be organized " + , " for each type of test. We do this to keep generality and " + , " allow support for tests the do not fit well in the " + , " 'function (input) == output' structure, like property " + , " tests. " + , " " + , " The only thing enforced regarding how test data should be " + , " structured is the error encoding, because it was agreed " + , " and it doesn't restrict flexibility in a significant way. " + , " " + , " Standardized property names may help when automatically " + , " deriving JSON parsers in some languages, so we followed " + , " a few conventions from the 'Google JSON Style Guide'. " + , " " + , " Additionally, this schema strictly enforces letters, in " + , " lowerCamelCase, for naming the 'property' being tested. We " + , " expect this regularity will allow an easier automatic " + , " generation of function's names in test generators, " + , " slightly reducing the amount of manually generated code. " + ], + + "$schema": "http://json-schema.org/draft-04/schema#", + + "self": { "vendor" : "io.exercism" + , "name" : "canonical-data" + , "format" : "jsonschema" + , "version": "1-0-0" + }, + + "$ref": "#/definitions/canonicalData", + + "definitions":{ + + "canonicalData": + { "description": "This is the top-level file structure" + , "type" : "object" + , "required" : ["exercise" , "version", "cases"] + , "properties" : + { "exercise" : { "$ref": "#/definitions/exercise" } + , "version" : { "$ref": "#/definitions/version" } + , "comments" : { "$ref": "#/definitions/comments" } + , "cases" : { "$ref": "#/definitions/testGroup" } + } + , "additionalProperties": false + }, + + "exercise": + { "description": "Exercise's slug (kebab-case)" + , "type" : "string" + , "pattern" : "^[a-z]+(-[a-z]+)*$" + }, + + "version" : + { "description" : "Semantic versioning: MAJOR.MINOR.PATCH" + , "type" : "string" + , "pattern" : "^(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*)){2}$" + }, + + "comments": + { "description": "An array of strings to fake multi-line comments" + , "type" : "array" + , "items" : { "type": "string" } + , "minItems" : 1 + }, + + "testGroup": + { "description": "An array of labeled test items" + , "type" : "array" + , "items" : { "$ref": "#/definitions/labeledTestItem" } + , "minItems" : 1 + }, + + "labeledTestItem": + { "description": "A single test or group of tests with a description" + , "oneOf": [ { "$ref": "#/definitions/labeledTest" } + , { "$ref": "#/definitions/labeledTestGroup" } + ] + }, + + "labeledTest": + { "description": "A single test with a description" + , "type" : "object" + , "required" : ["description", "property"] + , "properties" : + { "description": { "$ref": "#/definitions/description" } + , "comments" : { "$ref": "#/definitions/comments" } + , "property" : { "$ref": "#/definitions/property" } + , "expected" : { "$ref": "#/definitions/expected" } + } + }, + + "labeledTestGroup": + { "description": "A group of tests with a description" + , "type" : "object" + , "required" : ["description", "cases"] + , "properties" : + { "description": { "$ref": "#/definitions/description" } + , "comments" : { "$ref": "#/definitions/comments" } + , "cases" : { "$ref": "#/definitions/testGroup" } + } + , "additionalProperties": false + }, + + "description": + { "description": "A short, clear, one-line description" + , "type" : "string" + }, + + "property": + { "description": "A letters-only, lowerCamelCase property name" + , "type" : "string" + , "pattern" : "^[a-z]+([A-Z][a-z]+)*[A-Z]?$" + }, + + "expected": + { "description": "The expected return value of a test case" + , "properties": + { "error": { "$ref": "#/definitions/error" } + } + , "dependencies": + { "error": { "maxProperties": 1 } + } + }, + + "error": + { "description": "A message describing an error condition" + , "type" : "string" + } + + } + +}