Skip to content

Commit

Permalink
Merge pull request #1773 from matrix-org/travis/spec/rooms
Browse files Browse the repository at this point in the history
Add a room version specification
  • Loading branch information
turt2live authored Jan 24, 2019
2 parents 84a4ca6 + 061f595 commit 6c7eea5
Show file tree
Hide file tree
Showing 9 changed files with 447 additions and 245 deletions.
2 changes: 1 addition & 1 deletion scripts/gendoc.py
Original file line number Diff line number Diff line change
Expand Up @@ -457,7 +457,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions):

rst_file = os.path.join(tmp_dir, "spec_%s.rst" % (target_name,))
if version_label:
d = os.path.join(dest_dir, target_name)
d = os.path.join(dest_dir, target_name.split('@')[0])
if not os.path.exists(d):
os.mkdir(d)
html_file = os.path.join(d, "%s.html" % version_label)
Expand Down
34 changes: 15 additions & 19 deletions scripts/templating/matrix_templates/sections.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@
import inspect
import json
import os
import logging


logger = logging.getLogger(__name__)

class MatrixSections(Sections):

# pass through git ver so it'll be dropped in the input file
Expand All @@ -28,26 +31,19 @@ def render_git_version(self):
def render_git_rev(self):
return self.units.get("git_version")["revision"]

def render_client_server_changelog(self):
changelogs = self.units.get("changelogs")
return changelogs["client_server"]

# TODO: We should make this a generic variable instead of having to add functions all the time.
def render_push_gateway_changelog(self):
changelogs = self.units.get("changelogs")
return changelogs["push_gateway"]

def render_identity_service_changelog(self):
changelogs = self.units.get("changelogs")
return changelogs["identity_service"]

def render_server_server_changelog(self):
changelogs = self.units.get("changelogs")
return changelogs["server_server"]

def render_application_service_changelog(self):
def render_changelogs(self):
rendered = {}
changelogs = self.units.get("changelogs")
return changelogs["application_service"]
for spec, versioned in changelogs.items():
spec_var = "%s_changelog" % spec
logger.info("Rendering changelog for spec: %s" % spec)
for version, changelog in versioned.items():
version_var = "%s_%s" % (spec_var, version)
logger.info("Rendering changelog for %s" % version_var)
rendered[version_var] = changelog
if version == "unstable":
rendered[spec_var] = changelog
return rendered

def _render_events(self, filterFn, sortFn):
template = self.env.get_template("events.tmpl")
Expand Down
51 changes: 45 additions & 6 deletions scripts/templating/matrix_templates/units.py
Original file line number Diff line number Diff line change
Expand Up @@ -906,11 +906,26 @@ def read_event_schema(self, filepath):
def load_changelogs(self):
changelogs = {}

# Changelog generation is a bit complicated. We rely on towncrier to
# generate the unstable/current changelog, but otherwise use the RST
# edition to record historical changelogs. This is done by prepending
# the towncrier output to the RST in memory, then parsing the RST by
# hand. We parse the entire changelog to create a changelog for each
# version which may be of use in some APIs.

# Map specific headers to specific keys that'll be used eventually
# in variables. Things not listed here will get lowercased and formatted
# such that characters not [a-z0-9] will be replaced with an underscore.
keyword_versions = {
"Unreleased Changes": "unstable"
}

# Only generate changelogs for things that have an RST document
for f in os.listdir(CHANGELOG_DIR):
if not f.endswith(".rst"):
continue
path = os.path.join(CHANGELOG_DIR, f)
name = f[:-4]
name = f[:-4] # take off ".rst"

# If there's a directory with the same name, we'll try to generate
# a towncrier changelog and prepend it to the general changelog.
Expand Down Expand Up @@ -959,15 +974,39 @@ def load_changelogs(self):
prev_line = line
else: # have title, get body (stop on next title or EOF)
if re.match("^[=]{3,}$", line.strip()):
# we added the title in the previous iteration, pop it
# then bail out.
changelog_lines.pop()
break
# we hit another title, so pop the last line of
# the changelog and record the changelog
new_title = changelog_lines.pop()
if name not in changelogs:
changelogs[name] = {}
if title_part in keyword_versions:
title_part = keyword_versions[title_part]
title_part = title_part.strip().replace("^[a-zA-Z0-9]", "_").lower()
changelog = "".join(changelog_lines)
changelogs[name][title_part] = changelog

# reset for the next version
changelog_lines = []
title_part = new_title.strip()
continue
# Don't generate subheadings (we'll keep the title though)
if re.match("^[-]{3,}$", line.strip()):
continue
if line.strip().startswith(".. version: "):
# The changelog is directing us to use a different title
# for the changelog.
title_part = line.strip()[len(".. version: "):]
continue
if line.strip().startswith(".. "):
continue # skip comments
changelog_lines.append(" " + line + '\n')
changelogs[name] = "".join(changelog_lines)
if len(changelog_lines) > 0 and title_part is not None:
if name not in changelogs:
changelogs[name] = {}
if title_part in keyword_versions:
title_part = keyword_versions[title_part]
changelog = "".join(changelog_lines)
changelogs[name][title_part.replace("^[a-zA-Z0-9]", "_").lower()] = changelog

return changelogs

Expand Down
42 changes: 8 additions & 34 deletions specification/appendices/identifier_grammar.rst
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@
Identifier Grammar
------------------

Some identifiers are specific to given room versions, please refer to the
`room versions specification`_ for more information.

.. _`room versions specification`: ../index.html#room-versions


Server Name
~~~~~~~~~~~

Expand Down Expand Up @@ -78,38 +84,6 @@ Some recommendations for a choice of server name follow:
* The length of the complete server name should not exceed 230 characters.
* Server names should not use upper-case characters.


Room Versions
~~~~~~~~~~~~~

Room versions are used to change properties of rooms that may not be compatible
with other servers. For example, changing the rules for event authorization would
cause older servers to potentially end up in a split-brain situation due to them
not understanding the new rules.

A room version is defined as a string of characters which MUST NOT exceed 32
codepoints in length. Room versions MUST NOT be empty and SHOULD contain only
the characters ``a-z``, ``0-9``, ``.``, and ``-``.

Room versions are not intended to be parsed and should be treated as opaque
identifiers. Room versions consisting only of the characters ``0-9`` and ``.``
are reserved for future versions of the Matrix protocol.

The complete grammar for a legal room version is::

room_version = 1*room_version_char
room_version_char = DIGIT
/ %x61-7A ; a-z
/ "-" / "."

Examples of valid room versions are:

* ``1`` (would be reserved by the Matrix protocol)
* ``1.2`` (would be reserved by the Matrix protocol)
* ``1.2-beta``
* ``com.example.version``


Common Identifier Format
~~~~~~~~~~~~~~~~~~~~~~~~

Expand Down Expand Up @@ -327,7 +301,7 @@ matrix.to navigation

.. NOTE::
This namespacing is in place pending a ``matrix://`` (or similar) URI scheme.
This is **not** meant to be interpreted as an available web service - see
This is **not** meant to be interpreted as an available web service - see
below for more details.

Rooms, users, aliases, and groups may be represented as a "matrix.to" URI.
Expand All @@ -343,7 +317,7 @@ in RFC 3986:
The identifier may be a room ID, room alias, user ID, or group ID. The extra
parameter is only used in the case of permalinks where an event ID is referenced.
The matrix.to URI, when referenced, must always start with ``https://matrix.to/#/``
followed by the identifier.
followed by the identifier.

Clients should not rely on matrix.to URIs falling back to a web server if accessed
and instead should perform some sort of action within the client. For example, if
Expand Down
67 changes: 67 additions & 0 deletions specification/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,73 @@ dedicated API. The API is symmetrical to managing Profile data.
Would it really be overengineered to use the same API for both profile &
private user data, but with different ACLs?
Room Versions
-------------

Rooms are central to how Matrix operates, and have strict rules for what
is allowed to be contained within them. Rooms can also have various
algorithms that handle different tasks, such as what to do when two or
more events collide in the underlying DAG. To allow rooms to be improved
upon through new algorithms or rules, "room versions" are employed to
manage a set of expectations for each room. New room versions are assigned
as needed.

There is no implicit ordering or hierarchy to room versions, and their principles
are immutable once placed in the specification. Although there is a recommended
set of versions, some rooms may benefit from features introduced by other versions.
Rooms move between different versions by "upgrading" to the desired version. Due
to versions not being ordered or hierarchical, this means a room can "upgrade"
from version 2 to version 1, if it is so desired.

Room version grammar
~~~~~~~~~~~~~~~~~~~~

Room versions are used to change properties of rooms that may not be compatible
with other servers. For example, changing the rules for event authorization would
cause older servers to potentially end up in a split-brain situation due to not
understanding the new rules.

A room version is defined as a string of characters which MUST NOT exceed 32
codepoints in length. Room versions MUST NOT be empty and SHOULD contain only
the characters ``a-z``, ``0-9``, ``.``, and ``-``.

Room versions are not intended to be parsed and should be treated as opaque
identifiers. Room versions consisting only of the characters ``0-9`` and ``.``
are reserved for future versions of the Matrix protocol.

The complete grammar for a legal room version is::

room_version = 1*room_version_char
room_version_char = DIGIT
/ %x61-7A ; a-z
/ "-" / "."

Examples of valid room versions are:

* ``1`` (would be reserved by the Matrix protocol)
* ``1.2`` (would be reserved by the Matrix protocol)
* ``1.2-beta``
* ``com.example.version``

Complete list of room versions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Room versions are divided into two distinct groups: stable and unstable. Stable
room versions may be used by rooms safely. Unstable room versions are everything
else which is either not listed in the specification or flagged as unstable for
some other reason. Versions can switch between stable and unstable periodically
for a variety of reasons, including discovered security vulnerabilites and age.

Clients should not ask room administrators to upgrade their rooms if the room is
running a stable version. Servers SHOULD use room version 1 as the default room
version when creating new rooms.

The available room versions are:

* `Version 1 <rooms/v1.html>`_ - **Stable**. The current version of most rooms.
* `Version 2 <rooms/v2.html>`_ - **Stable**. Implements State Resolution Version 2.

Specification Versions
----------------------

Expand Down
98 changes: 98 additions & 0 deletions specification/rooms/v1.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
.. Copyright 2017,2019 New Vector Ltd
..
.. Licensed under the Apache License, Version 2.0 (the "License");
.. you may not use this file except in compliance with the License.
.. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
.. Unless required by applicable law or agreed to in writing, software
.. distributed under the License is distributed on an "AS IS" BASIS,
.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.. See the License for the specific language governing permissions and
.. limitations under the License.
Room Version 1
==============

This room version is the first ever version for rooms, and contains the building
blocks for other room versions.

Server implementation components
--------------------------------

.. WARNING::
The information contained in this section is strictly for server implementors.
Applications which use the Client-Server API are generally unaffected by the
details contained here, and can safely ignore their presence.


The algorithms defined here should only apply to version 1 rooms. Other algorithms
may be used by other room versions, and as such servers should be aware of which
version room they are dealing with prior to executing a given algorithm.

.. WARNING::
Although room version 1 is the most popular room version, it is known to have
undesirable effects. Servers implementing support for room version 1 should be
aware that restrictions should be generally relaxed and that inconsistencies
may occur until room version 2 (or later) is ready and adopted.

State resolution
~~~~~~~~~~~~~~~~

.. WARNING::
This section documents the state resolution algorithm as implemented by
Synapse as of December 2017 (and therefore the de-facto Matrix protocol).
However, this algorithm is known to have some problems.

The room state :math:`S'(E)` after an event :math:`E` is defined in terms of
the room state :math:`S(E)` before :math:`E`, and depends on whether
:math:`E` is a state event or a message event:

* If :math:`E` is a message event, then :math:`S'(E) = S(E)`.

* If :math:`E` is a state event, then :math:`S'(E)` is :math:`S(E)`, except
that its entry corresponding to :math:`E`'s ``event_type`` and ``state_key``
is replaced by :math:`E`'s ``event_id``.

The room state :math:`S(E)` before :math:`E` is the *resolution* of the set of
states :math:`\{ S'(E'), S'(E''), … \}` consisting of the states after each of
:math:`E`'s ``prev_event``\s :math:`\{ E', E'', … \}`.

The *resolution* of a set of states is defined as follows. The resolved state
is built up in a number of passes; here we use :math:`R` to refer to the
results of the resolution so far.

* Start by setting :math:`R` to the union of the states to be resolved,
excluding any *conflicting* events.

* First we resolve conflicts between ``m.room.power_levels`` events. If there
is no conflict, this step is skipped, otherwise:

* Assemble all the ``m.room.power_levels`` events from the states to
be resolved into a list.

* Sort the list by ascending ``depth`` then descending ``sha1(event_id)``.

* Add the first event in the list to :math:`R`.

* For each subsequent event in the list, check that the event would be
allowed by the `authorization rules`_ for a room in state :math:`R`. If the
event would be allowed, then update :math:`R` with the event and continue
with the next event in the list. If it would not be allowed, stop and
continue below with ``m.room.join_rules`` events.

* Repeat the above process for conflicts between ``m.room.join_rules`` events.

* Repeat the above process for conflicts between ``m.room.member`` events.

* No other events affect the authorization rules, so for all other conflicts,
just pick the event with the highest depth and lowest ``sha1(event_id)`` that
passes authentication in :math:`R` and add it to :math:`R`.

A *conflict* occurs between states where those states have different
``event_ids`` for the same ``(state_type, state_key)``. The events thus
affected are said to be *conflicting* events.


.. _`authorization rules`: ../server_server/unstable.html#authorization-rules
Loading

0 comments on commit 6c7eea5

Please sign in to comment.