Merge pull request #152 from FDSN/draft

Version 1.2

Changes.md

@@ -1,5 +1,10 @@
 # Changes for FDSN StationXML format
 
+## Changes from version 1.1 to 1.2 (2022-02-25)
+
+Version 1.2 introduces no schema "as a schema" changes, other than the
+schemaVersion attribute. All changes are additions of annotations and other documentation.
+
 ## Changes from version 1.0 to 1.1 (2019-5-3)
 
 (Edited 2019-12-18 for small clarifications)
@@ -31,4 +36,3 @@
 * Add `<WaterLevel>` within `<Station>` and `<Channel>`
 
 * Add "subject" attribute to `<Comment>` to allow relating comments, make "id" attribute optional.
-

README.md

@@ -35,3 +35,13 @@ described above, must be followed for any changes to the schema.
 
 Changes and issues should only be grouped together when logically
 related in order to streamline review and acceptance.
+
+## Translating StationXML 1.0 to 1.1 and later releases
+
+The vast majority of the StationXML 1.0 schema exists in the 1.1 schema, making most 1.0 documents compatible
+with the 1.1 schema.  There are a few small exceptions where 1.0 elements were removed from 1.1, in one
+important case to avoid the specification of incorrect metadata.
+
+An [XSLT definition for StationXML 1.0 to 1.1 conversion](StationXML-1.0to1.1.xslt) exists to assist
+with the systematic conversion of version 1.0 documents to the version 1.1 schema.  This is done by removing the
+elements no longer allowed in 1.1. A similar [XSLT for version 1.0/1.1 to 1.2](StationXML-1.0to1.2.xslt) also exists.

StationXML-1.0to1.1.xslt

@@ -0,0 +1,39 @@
+<xsl:transform version="1.0"
+ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:fsx="http://www.fdsn.org/xml/station/1">
+
+    <!--
+    This XSLT 1.0 transform may be used to translate a StationXML version 1.0 document to version 1.1
+    Version: 2020-04-07
+
+    The document *must* declare the FDSN StationXML elements in a namespace of "http://www.fdsn.org/xml/station/1"
+
+    The following changes are applied by the transformation:
+    1) Change FDSNStationXML@schemaVersion from 1.0 to 1.1
+    2) Remove any .../Channel/StorageFormat elements, which are not allowed in 1.1
+    3) Remove any .../Channel/Response/Stage/StageGain elements when
+                  .../Channel/Response/Stage/Polynomial elements are present,
+         which are not allowed in 1.1 as they are nearly always incompatible.
+         Note: This tranformation assumes the StageGain elements are in the document by
+         error (to stricly conform to 1.0) and the Polynomial is the intended filter.
+    -->
+
+    <!-- Identity template: copy all input to output and apply the included templates -->
+    <xsl:template match="node() | @*">
+      <xsl:copy>
+         <xsl:apply-templates select="node() | @*"/>
+      </xsl:copy>
+    </xsl:template>
+
+    <!-- Change schemaVersion from 1.0 to 1.1 -->
+    <xsl:template match="/fsx:FDSNStationXML/@schemaVersion[. = '1.0']">
+      <xsl:attribute name="schemaVersion">1.1</xsl:attribute>
+    </xsl:template>
+
+    <!-- Remove StorageFormat -->
+    <xsl:template match="/fsx:FDSNStationXML/fsx:Network/fsx:Station/fsx:Channel/fsx:StorageFormat"/>
+
+    <!-- Remove StageGain element from Stage that contains Polynomial -->
+    <xsl:template match="/fsx:FDSNStationXML/fsx:Network/fsx:Station/fsx:Channel/fsx:Response/fsx:Stage[fsx:Polynomial]/fsx:StageGain"/>
+
+</xsl:transform>

StationXML-1.0to1.2.xslt

@@ -0,0 +1,46 @@
+<xsl:transform version="1.0"
+ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:fsx="http://www.fdsn.org/xml/station/1">
+
+    <!--
+    This XSLT 1.0 transform may be used to translate a StationXML
+    version 1.0 or 1.1 document to version 1.2
+    Version: 2022-02-25
+
+    The document *must* declare the FDSN StationXML elements in a namespace of
+    "http://www.fdsn.org/xml/station/1"
+
+    The following changes are applied by the transformation:
+    1) Change FDSNStationXML@schemaVersion from 1.0 to 1.2
+    2) Remove any .../Channel/StorageFormat elements, which are not allowed in 1.1
+    3) Remove any .../Channel/Response/Stage/StageGain elements when
+                  .../Channel/Response/Stage/Polynomial elements are present,
+         which are not allowed in 1.1 as they are nearly always incompatible.
+         Note: This tranformation assumes the StageGain elements are in the document by
+         error (to stricly conform to 1.0) and the Polynomial is the intended filter.
+    -->
+
+    <!-- Identity template: copy all input to output and apply the included templates -->
+    <xsl:template match="node() | @*">
+      <xsl:copy>
+         <xsl:apply-templates select="node() | @*"/>
+      </xsl:copy>
+    </xsl:template>
+
+    <!-- Change schemaVersion from 1.0 to 1.2 -->
+    <xsl:template match="/fsx:FDSNStationXML/@schemaVersion[. = '1.0']">
+      <xsl:attribute name="schemaVersion">1.2</xsl:attribute>
+    </xsl:template>
+
+    <!-- Also change schemaVersion from 1.1 to 1.2 -->
+    <xsl:template match="/fsx:FDSNStationXML/@schemaVersion[. = '1.1']">
+      <xsl:attribute name="schemaVersion">1.2</xsl:attribute>
+    </xsl:template>
+
+    <!-- Remove StorageFormat -->
+    <xsl:template match="/fsx:FDSNStationXML/fsx:Network/fsx:Station/fsx:Channel/fsx:StorageFormat"/>
+
+    <!-- Remove StageGain element from Stage that contains Polynomial -->
+    <xsl:template match="/fsx:FDSNStationXML/fsx:Network/fsx:Station/fsx:Channel/fsx:Response/fsx:Stage[fsx:Polynomial]/fsx:StageGain"/>
+
+</xsl:transform>

auditDocUpdate/.gitignore

@@ -0,0 +1,3 @@
+fdsn-station-1.1.xsd
+fdsn-station*.nodocs
+fdsn-station*.nodocs.tidy

auditDocUpdate/README.md

@@ -0,0 +1,38 @@
+### Audit Documentation Changes
+
+Scripts to ensure addition of documentation does not make changes to the
+structure of the FDSN StationXML schema. Running compare.sh should produce no
+differences if the changes are confined to the documentation.
+
+transform.py uses nodocs.xslt to remove all annotations, then tidy
+cleans the output to remove non-structural changes like indentation.
+Finally diff shows the raw textual differences between the two versions.
+
+## Install
+
+lxml is used by transform.py, install with:
+
+```
+conda install lxml
+```
+
+# Run
+
+```
+./compare.sh
+```
+
+If there are no schema changes besides annotations, the output should look like:
+
+```
+% ./compare.sh
+Using: fdsn-station-1.1.xsd
+Writing transformed XML to fdsn-station-1.1.xsd.nodocs
+Writing transformed XML to fdsn-station-local.xsd.nodocs
+diff from fdsn-station-local.xsd to fdsn-station-1.1.xsd
+< is local tidy version, fdsn-station-local.xsd
+> is fdsn tidy version, fdsn-station-1.1.xsd
+Note line numbers are in nodocs.tidy,
+which will be different from original xsd files.
+
+```

auditDocUpdate/check_ascii.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+import sys
+import re
+
+opts = [opt for opt in sys.argv[1:] if opt.startswith("-")]
+args = [arg for arg in sys.argv[1:] if not arg.startswith("-")]
+
+if len(args) == 0:
+  print (f'Usage: {sys.argv[0]} inputXML ')
+  exit(0)
+
+# Load source document
+with open(args[0],'r') as doc:
+    for line_no, line in enumerate(doc):
+        non_ascii = re.sub('[\x00-\x7f]', '', line)
+        if len(non_ascii) > 0:
+            print(f"Found non-ascii chars of {non_ascii} on line {line_no}")
+
+

auditDocUpdate/compare.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# exit on error
+set -e
+
+XSD=fdsn-station-1.1.xsd
+echo Using: $XSD
+
+if [[ ! -f "$XSD" ]]; then
+  wget http://www.fdsn.org/xml/station/${XSD}
+fi
+
+/bin/rm -f ${XSD}.nodocs.tidy ${XSD}.nodocs
+python3 ./transform.py $XSD nodocs.xslt ${XSD}.nodocs
+tidy -xml -iq  -o ${XSD}.nodocs.tidy ${XSD}.nodocs
+/bin/rm -f ${XSD}.nodocs
+
+LOCALXSD=fdsn-station-local.xsd
+/bin/rm -f ${LOCALXSD}.nodocs ${LOCALXSD}.nodocs.tidy
+python3 ./transform.py ../fdsn-station.xsd nodocs.xslt ${LOCALXSD}.nodocs
+tidy -xml -iq -o ${LOCALXSD}.nodocs.tidy ${LOCALXSD}.nodocs
+/bin/rm -f ${LOCALXSD}.nodocs
+
+echo diff from ${LOCALXSD} to ${XSD}
+echo '<' is local tidy version, ${LOCALXSD}
+echo '>' is fdsn tidy version, ${XSD}
+echo Note line numbers are in nodocs.tidy,
+echo which will be different from original xsd files.
+echo
+
+diff -b ${LOCALXSD}.nodocs.tidy ${XSD}.nodocs.tidy

auditDocUpdate/nodocs.xslt

@@ -0,0 +1,26 @@
+<xsl:transform version="1.0"
+ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema">
+
+    <!--
+    This XSLT 1.0 transform may be used to remove annotation/documentation elements from XML
+
+    Annotation/Documentation elements stripped are identified as:
+    xmlns:xs="http://www.w3.org/2001/XMLSchema"
+    xs:annotation
+    -->
+
+    <!-- Identity template: copy all input to output and apply the included templates -->
+    <xsl:template match="node() | @*">
+      <xsl:copy>
+         <xsl:apply-templates select="node() | @*"/>
+      </xsl:copy>
+    </xsl:template>
+
+    <!-- Remove annotations -->
+    <xsl:template match="xs:annotation"/>
+
+    <!-- Remove comments (like this one) -->
+    <xsl:template match="comment()"/>
+
+</xsl:transform>

auditDocUpdate/transform.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+from lxml import etree
+import sys
+
+opts = [opt for opt in sys.argv[1:] if opt.startswith("-")]
+args = [arg for arg in sys.argv[1:] if not arg.startswith("-")]
+
+if len(args) == 0:
+  print (f'Usage: {sys.argv[0]} inputXML [XSLTfile [outputXML]]')
+  exit(0)
+
+# Load source document
+doc = etree.parse(args[0])
+
+# Perform XSLT tranform
+if len(args) > 1:
+    xslt = etree.parse(args[1])
+    transform = etree.XSLT(xslt)
+    newdoc = transform(doc)
+
+    # force end tags to make for easier diff
+
+    for elem in newdoc.iter(tag=etree.Element):
+        if elem.text is not None:
+            elem.text = elem.text.strip()
+        if elem.text is None or len(elem.text) == 0:
+            elem.text = '\n'
+
+    # Write to output file
+    if len(args) > 2:
+        print (f'Writing transformed XML to {args[2]}')
+        newdoc.write(args[2], pretty_print=False, strip_text=True)
+        #newdoc.write_output(args[2]) #, pretty_print=False)

docs/.gitignore

@@ -0,0 +1,2 @@
+_build
+__pycache__/

docs/Makefile

@@ -0,0 +1,29 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+SCHEMA_FILE   = $(SOURCEDIR)/../fdsn-station.xsd
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# convert generates: level-network.rst level-station.rst level-channel.rst
+#                    level-response.rst warnings.rst
+convert:
+	python convert_xsd_to_rst.py "$(SCHEMA_FILE)" "$(SOURCEDIR)"
+
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile 
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+