diff --git a/.gitignore b/.gitignore index 72d5f9267de..e3f9640d0a2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,10 @@ bin # Misspell binary internal/tools/bin +.tools + +# Pytest cache +__pycache__ + +# JSON files generated by the specification parser +*.json diff --git a/.markdownlint.yaml b/.markdownlint.yaml index d2a0d946e79..8010e6ef5d6 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -1,6 +1,9 @@ # Default state for all rules default: true +# heading-increment" +MD001: false + # ul-style MD004: false @@ -15,9 +18,14 @@ MD024: MD029: style: ordered +# no-blanks-blockquote +MD028: false + # no-inline-html MD033: false +# no-space-in-code +MD038: false + # fenced-code-language MD040: false - diff --git a/README.md b/README.md index 4010397e86e..879498a178f 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Technical committee holds regular meetings, notes are held ## Table of Contents - [Overview](specification/overview.md) +- [Requirements](specification/requirement-sections.md) - [Glossary](specification/glossary.md) - [Versioning and stability for OpenTelemetry clients](specification/versioning-and-stability.md) - [Library Guidelines](specification/library-guidelines.md) diff --git a/internal/tools/specification_parser/specification_parser.py b/internal/tools/specification_parser/specification_parser.py new file mode 100644 index 00000000000..da10416042a --- /dev/null +++ b/internal/tools/specification_parser/specification_parser.py @@ -0,0 +1,129 @@ +from re import finditer, findall +from json import dumps +from os.path import curdir, abspath, join, splitext +from os import walk + + +def find_markdown_file_paths(root): + markdown_file_paths = [] + + for root_path, _, file_paths, in walk(root): + for file_path in file_paths: + absolute_file_path = join(root_path, file_path) + + _, file_extension = splitext(absolute_file_path) + + if file_extension == ".md": + markdown_file_paths.append(absolute_file_path) + + return markdown_file_paths + + +def parse_requirements(markdown_file_paths): + requirements = {} + + for markdown_file_path in markdown_file_paths: + + with open(markdown_file_path, "r") as markdown_file: + + text = markdown_file.read() + + requirement_matches = [ + requirement_match.groupdict() for requirement_match in ( + finditer( + r"###### requirement:\s(?P[_\w]+)\n\n" + r"(?P(>.*\n)+)", + text, + ) + ) + ] + + if not requirement_matches: + continue + + md_file_path = "".join([splitext(markdown_file_path)[0], ".md"]) + + requirements[md_file_path] = {} + + for requirement in requirement_matches: + + requirement_key = requirement["key"] + + assert ( + requirement_key not in + requirements[md_file_path].keys() + ), "Repeated requirement key {} found in {}".format( + requirement_key, markdown_file_path + ) + + requirement_description = requirement["description"].strip() + + BCP_14_keyword_matches = [] + + for BCP_14_keyword_regex in [ + # 2. MUST NOT + r"MUST NOT", + r"SHALL NOT", + # 1. MUST + r"MUST(?! NOT)", + r"REQUIRED", + r"SHALL(?! NOT)", + # 4. SHOULD NOT + r"SHOULD NOT", + r"NOT RECOMMENDED", + # 3. SHOULD + r"SHOULD(?! NOT)", + r"(? This MUST be done.", + "BCP 14 Keywords": ["MUST"] + } + + def test_parseable_section_1(self): + assert self.parsed_requirements["parseable_section_1"] == { + "description": "> This MUST NOT be done.", + "BCP 14 Keywords": ["MUST NOT"] + } + + def test_parseable_section_2(self): + assert self.parsed_requirements["parseable_section_2"] == { + "description": "> This SHOULD be done.", + "BCP 14 Keywords": ["SHOULD"] + } + + def test_parseable_section_3(self): + assert self.parsed_requirements["parseable_section_3"] == { + "description": "> This SHOULD NOT be done.", + "BCP 14 Keywords": ["SHOULD NOT"] + } + + def test_parseable_section_4(self): + assert self.parsed_requirements["parseable_section_4"] == { + "description": "> This MAY be done.", + "BCP 14 Keywords": ["MAY"] + } + + def test_parseable_section_5(self): + assert self.parsed_requirements["parseable_section_5"] == { + "description": "> This **MAY** be done 5.", + "BCP 14 Keywords": ["MAY"] + } + + def test_parseable_section_6(self): + assert self.parsed_requirements["parseable_section_6"] == { + "description": "> This *MAY* be done 6.", + "BCP 14 Keywords": ["MAY"] + } + + def test_parseable_section_7(self): + assert self.parsed_requirements["parseable_section_7"] == { + "description": ( + "> This *MAY* be done 7.\n> This is section 7." + ), + "BCP 14 Keywords": ["MAY"] + } + + def test_parseable_section_8(self): + assert self.parsed_requirements["parseable_section_8"] == { + "description": "> This *MAY* be done 8.\n>\n> This is section 8.", + "BCP 14 Keywords": ["MAY"] + } + + def test_parseable_section_9(self): + assert self.parsed_requirements["parseable_section_9"] == { + "description": ( + "> This *MAY* be done 9.\n>\n> This is section 9.\n>\n" + "> 1. Item 1\n> 2. Item 2\n> 1. Item 2.1\n" + "> 2. Item 2.2" + ), + "BCP 14 Keywords": ["MAY"] + } + + def test_parseable_section_10(self): + assert self.parsed_requirements["parseable_section_10"] == { + "description": ( + "> This *MAY* be done 10\n>\n> If this is done, then this " + "MUST be that and MUST NOT be\n> something else." + ), + "BCP 14 Keywords": ["MAY", "MUST", "MUST NOT"] + } diff --git a/internal/tools/specification_parser/test_specification.md b/internal/tools/specification_parser/test_specification.md new file mode 100644 index 00000000000..eea18cff225 --- /dev/null +++ b/internal/tools/specification_parser/test_specification.md @@ -0,0 +1,62 @@ +# Test Specification + +## Title + +Some content. + +###### requirement: parseable_section_0 + +> This MUST be done. + +###### requirement: parseable_section_1 + +> This MUST NOT be done. + +###### requirement: parseable_section_2 + +> This SHOULD be done. + +###### requirement: parseable_section_3 + +> This SHOULD NOT be done. + +###### requirement: parseable_section_4 + +> This MAY be done. + +###### requirement: parseable_section_5 + +> This **MAY** be done 5. + +###### requirement: parseable_section_6 + +> This *MAY* be done 6. + +###### requirement: parseable_section_7 + +> This *MAY* be done 7. +> This is section 7. + +###### requirement: parseable_section_8 + +> This *MAY* be done 8. +> +> This is section 8. + +###### requirement: parseable_section_9 + +> This *MAY* be done 9. +> +> This is section 9. +> +> 1. Item 1 +> 2. Item 2 +> 1. Item 2.1 +> 2. Item 2.2 + +###### requirement: parseable_section_10 + +> This *MAY* be done 10 +> +> If this is done, then this MUST be that and MUST NOT be +> something else. diff --git a/specification/metrics/api.md b/specification/metrics/api.md index ae0e397bb63..a2b8d1d4203 100644 --- a/specification/metrics/api.md +++ b/specification/metrics/api.md @@ -108,12 +108,23 @@ can be configured at run time. ### Behavior of the API in the absence of an installed SDK -In the absence of an installed Metrics SDK, the Metrics API MUST consist only -of no-ops. None of the calls on any part of the API can have any side effects -or do anything meaningful. Meters MUST return no-op implementations of any -instruments. From a user's perspective, calls to these should be ignored without raising errors -(i.e., *no* `null` references MUST be returned in languages where accessing these results in errors). -The API MUST NOT throw exceptions on any calls made to it. +###### requirement: only_no_ops + +> The metrics API MUST consist only of no-ops. None of the calls on any part of the API can have any side effects or do anything meaningful. + +###### requirement: meters_return_no_ops + +> Meters MUST return no-op implementations of any instruments. + +From a user's perspective, calls to these should be ignored without raising errors. + +###### requirement: no_null_references + +> Null references MUST NOT be returned in languages where accessing these results in errors. + +###### requirement: no_api_exceptions + +> The API MUST NOT throw exceptions on any calls made to it. ### Measurements diff --git a/specification/requirement-sections.md b/specification/requirement-sections.md new file mode 100644 index 00000000000..e2377ba4007 --- /dev/null +++ b/specification/requirement-sections.md @@ -0,0 +1,139 @@ +# Requirement Sections + +
+ +Table of Contents + + + + + +- [Requirement Sections](#requirement-sections) + * [Requirement Sections Format](#requirement-sections-format) + + [Key Format](#key-format) + + [Description Format](#description-format) + + [Example](#example) + * [Purpose of the requirement sections](#purpose-of-the-requirement-sections) + + + +
+ +This document explains how the OpenTelemetry specification requirement sections are written. + +## Overview + +The OpenTelemetry specification is written in several [Markdown](https://github.github.com/gfm/) files. + +Each one of these files can contain any resource to explain and describe its part of the specification. +Examples of these resources are images, diagrams, code, regular text, etc. + +Also, included in the same Markdown documents that make the OpenTelemetry specification +are to be included specific sections named _requirement sections_ that follow a specific +format. These sections are the part of the document that more formally describes each of +the specific requirements that the OpenTelemetry specification has. + +Each of these requirement sections has 2 components: + +1. A unique **key**, a string that identifies the requirement section in a Markdown document +2. A **description**, a string that MUST include at least one of the [BCP 14 keywords](https://tools.ietf.org/html/bcp14). + +### Requirement Sections Format + +Each one of these requirement sections are written also in Markdown syntax in order for them to integrate +with the rest of the document. + +#### Key Format + +The key of every requirement section MUST be unique in the document that contains it. This key +MUST be written in this manner: + +``` +###### requirement: unique_key_identifier +``` + +The first six `#` symbols create a Markdown heading for the requirement section. The `requirement: ` +string that follows indicates that this particular header is part of a requirement section and not +just any Markdown six `#` level heading. The following string indicated by `unique_key_indetifier` MUST be +unique in the document that contains it. The characters that make this string MUST only be +alphanumeric characters and underscores. + +#### Description Format + +The description of every requirement section MUST be written as a +[block quote](https://github.github.com/gfm/#block-quotes) immediately following a blank line after a requirement section key: + +``` +> The span MUST have an identifier. +> +> More text can be placed here as well. +``` + +The description MUST include at least one BCP 14 keyword. + +#### Example + +Here is a small example that shows how a Markdown document can have requirement sections to describe +its specific requirements: + +``` +# Some title for some OpenTelemetry concept + +This part describes some OpenTelemetry concept. It can include examples, images, diagrams, etc. + +After the concept is described, its specifc requirements are written in requirement sections: + +###### requirement: concept_identifier + +> The concept MUST have an identifier. +> +> The concept is important for OpenTelemetry. + +###### requirement: concept_documentation + +> The concept SHOULD be documented in every implementation. +``` + +### Purpose of the requirement sections + +The idea behind writing the requirements in this manner is to make it easy for the reader to find all the +requirements included in an OpenTelemetry specification document. In this way, it is also easy to find all +the requirements a certain implementation must comply with. With all the requirements available for the +implementation developer, it is easy to make a list of test cases, one for every requirement section, and +to test the implementation against these test cases to measure compliance with the specification. This is +why the key must be unique, so that it can be used to form a name for the particular testing function for +that requirement. + +With the requirements specified in this way, it is also easier for the specification and implementation +developers to refer to a certain requirement unequivocally, making communication between developers more +clear. + +It is also possible to parse the Markdown documents and extract from them a list of the requirements in a +certain format. A parser that does this is provided. Itproduces JSON documents for every Markdown document +that includes at least one requirement section. With these JSON files, a testing schema can be produced for +every implementation that can help developers know how compliant with the specification the implementation is. + +The parser can also work as a checker that makes sure that every requirement section is compliant with this +specification. This can even be incorporated to the CI of the repo where the OpenTelemetry specification is +in order to reject any change that adds a non-compliant requirement section. + +Finally, it makes the specification developer follow a "testing mindset" while writing requirements. For example, +when writing a requirement, the specification developers ask themselves "can a test be written for this statement?". +This helps writing short, concise requirements that are clear for the implementation developers. + +### Running the specification parser + +The included specification parser can be run from the root directory of the OpenTelemetry specification directory +like this: + +``` +python internal/tools/specification_parser/specification_parser.py +``` + +This will recursively look for Markdown files in the `specification` directory. For every Markdown file that has at +least one requirement section, it will generate a corresponding JSON file with the key, description and BCP 14 +keyword or every requirement section. + +Once the JSON files are generated, they can be used by implementations as checklists to write test cases. These +test cases then are written to implement what is said in the description of each item in the JSON file. This set of +test cases can be used to measure how compliant with the specification an implementation is.