Fix wrong symbol defined in input excel for Q27 and the newton metre …

…per metre definition. (#7)
quadient · Nov 9, 2023 · 529f475 · 529f475
1 parent f378301
commit 529f475
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 47 deletions.
diff --git a/.run/parser.run.xml b/.run/parser.run.xml
@@ -7,11 +7,11 @@
       <env name="PYTHONUNBUFFERED" value="1" />
     </envs>
     <option name="SDK_HOME" value="" />
-    <option name="WORKING_DIRECTORY" value="C:\repos\quadient\unece-units\python\src\unece_excel_parser" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/python/src/unece_excel_parser" />
     <option name="IS_MODULE_SDK" value="true" />
     <option name="ADD_CONTENT_ROOTS" value="true" />
     <option name="ADD_SOURCE_ROOTS" value="true" />
-    <option name="SCRIPT_NAME" value="$PROJECT_DIR$/python/src/unece_excel_parser/unece_excel_parser.py" />
+    <option name="SCRIPT_NAME" value="unece_excel_parser.py" />
     <option name="PARAMETERS" value="-rec20 files/rec20_Rev17e-2021.xlsx -rec21 files/rec21_Rev12e_Annex-V-VI_2021.xls" />
     <option name="SHOW_COMMAND_LINE" value="false" />
     <option name="EMULATE_TERMINAL" value="false" />

diff --git a/README.md b/README.md
@@ -5,13 +5,14 @@ Units library supporting UNECE rec. 20 and rec. 21 standards
 # Development
 
 - developed on python `3.12.0`
+- mainly using `pint` library for linking units
 - in the project root folder:
-  - `cd python`
-  - create virtual environment `python -m venv venv`
-  - activate virtual environment, for example on windows run `.\venv\Scripts\Activate.ps1`
-  - run `pip install -e src/unece_excel_parser` to install dependencies & fix imports
-  - run `pip install -e src/unece_excel_parser[dev]` to install dev dependencies (for running tests)
-  - (optional) set python interpreter in dev IDE to point to python.exe in venv folder
+    - `cd python`
+    - create virtual environment `python -m venv venv`
+    - activate virtual environment, for example on windows run `.\venv\Scripts\Activate.ps1`
+    - run `pip install -e src/unece_excel_parser` to install dependencies & fix imports
+    - run `pip install -e src/unece_excel_parser[dev]` to install dev dependencies (for running tests)
+    - (optional) set python interpreter in dev IDE to point to python.exe in venv folder
 
 ## Run
 
@@ -24,4 +25,13 @@ From `python/src/unece_excel_parser` folder run:
 UNECE standards are located on https://unece.org/trade/uncefact/cl-recommendations. Basic description of the UNECE
 standard is located on https://unece.org/sites/default/files/2023-10/Rec20_Rev6e_2009.pdf.
 
-Also the Excel files and the PDF explaining the data in the standard are located in the `python/src/files` folder.
+Also the Excel files and the PDF explaining the data in the standard are located in the `python/src/files` folder.
+
+# Known issues
+
+The `pint` library does reduce the units and this could cause some unexpected
+behaviour where units get reduced, see https://github.com/hgrecco/pint/issues/551. For example `N * m / m` is reduced
+to `N`.
+
+The input excel for rec. 20 standard contains a lot of bugs, inconsistencies and typos and we are trying to correct
+them, but it could be not perfect.
diff --git a/python/src/unece_excel_parser/lib/pint_registry.py b/python/src/unece_excel_parser/lib/pint_registry.py
@@ -26,5 +26,6 @@ def __create_registry() -> UnitRegistry:
         registry.define('parts_per_billion_US = 1 ppb = ppb')
         registry.define("beats_per_minute = 1 BPM = BPM")
         registry.define("month = 1 mo = mo")
+        registry.define("Nm = newton * meter = N * m")
         registry.case_sensitive = True
         return registry
diff --git a/python/src/unece_excel_parser/lib/unit.py b/python/src/unece_excel_parser/lib/unit.py
@@ -1,13 +1,16 @@
-from pint import errors
-
-from lib.category import Category
+from lib.category import Category
 from lib.conversion_factor import ConversionFactor
 from lib.normalizer import Normalizer
 from lib.pint_registry import PintRegistryManager
 from lib.state import State
+from pint import errors
 
 
 class Unit:
+    _symbol_corrections_by_common_code: dict[(str, str | None)] = {
+        "Q27": "(N * m) / m",
+    }
+
     def __init__(self, state: State, common_code: str, name: str, description: str, categories: Category, symbol: str,
                  parsed_symbol: str, conversion_factor: ConversionFactor):
         self.state = state
@@ -50,11 +53,14 @@ def parse(state, common_code, name, description, category, symbol, conversion_fa
                     Normalizer.normalize_unit_name(name),
                     Normalizer.normalize_value(description),
                     Category.parse_categories(Normalizer.normalize_value(category)),
-                    symbol, Unit.__parse_unit_reference(symbol),
+                    symbol, Unit.__parse_unit_reference(symbol, common_code),
                     ConversionFactor.parse(conversion_factor))
 
     @staticmethod
-    def __parse_unit_reference(symbol: str) -> str | None:
+    def __parse_unit_reference(symbol: str, common_code: str) -> str | None:
+        if common_code in Unit._symbol_corrections_by_common_code:
+            symbol = Unit._symbol_corrections_by_common_code[common_code]
+
         if symbol is None:
             return None
 

diff --git a/python/src/unece_excel_parser/parsedUneceUnits.json b/python/src/unece_excel_parser/parsedUneceUnits.json
@@ -1556,8 +1556,7 @@
       "factor": 1.0,
       "parsedSymbol": "newton / meter",
       "commonCodeReferences": [
-        "M34",
-        "Q27"
+        "M34"
       ]
     },
     "parsedSymbol": "newton / meter",
@@ -4010,7 +4009,8 @@
       "factor": 9.80665,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": "force_kilogram",
@@ -4180,7 +4180,8 @@
       "factor": 1000,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": "kilonewton",
@@ -4254,7 +4255,8 @@
       "factor": 9.80665,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": null,
@@ -4608,7 +4610,8 @@
       "factor": 1000000,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": "meganewton",
@@ -4924,7 +4927,8 @@
       "factor": 1e-06,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": "micronewton",
@@ -5560,7 +5564,8 @@
       "factor": 0.001,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": "millinewton",
@@ -5587,8 +5592,7 @@
       "parsedSymbol": "newton / meter",
       "commonCodeReferences": [
         "4P",
-        "M34",
-        "Q27"
+        "M34"
       ]
     },
     "parsedSymbol": "millinewton / meter",
@@ -6491,7 +6495,8 @@
       "factor": 4.448222,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": "force_pound",
@@ -9035,7 +9040,8 @@
       "factor": 1e-05,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": "dyne",
@@ -9068,8 +9074,7 @@
       "parsedSymbol": "newton / meter",
       "commonCodeReferences": [
         "4P",
-        "M34",
-        "Q27"
+        "M34"
       ]
     },
     "parsedSymbol": "dyne / centimeter",
@@ -11588,7 +11593,7 @@
       "parsedSymbol": "kilogram * meter ** 2 / radian / second ** 2",
       "commonCodeReferences": null
     },
-    "parsedSymbol": "number_meter / degree",
+    "parsedSymbol": "Nm / degree",
     "categories": "LEVEL_1_NORMATIVE|LEVEL_1_NORMATIVE_COMMON"
   },
   {
@@ -19971,7 +19976,8 @@
       "factor": 0.2780139,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": "force_ounce",
@@ -20829,7 +20835,8 @@
       "factor": 8896.443,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": null,
@@ -21453,8 +21460,7 @@
       "parsedSymbol": "newton / meter",
       "commonCodeReferences": [
         "4P",
-        "M34",
-        "Q27"
+        "M34"
       ]
     },
     "parsedSymbol": "newton / centimeter",
@@ -21604,8 +21610,7 @@
       "factor": 1.0,
       "parsedSymbol": "newton / meter",
       "commonCodeReferences": [
-        "4P",
-        "Q27"
+        "4P"
       ]
     },
     "parsedSymbol": "newton / meter",
@@ -22286,7 +22291,8 @@
       "factor": 4448.222000000001,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": "kip",
@@ -22302,7 +22308,8 @@
       "factor": 0.138255,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": "poundal",
@@ -22332,7 +22339,8 @@
       "factor": 0.00980665,
       "parsedSymbol": "newton",
       "commonCodeReferences": [
-        "NEW"
+        "NEW",
+        "Q27"
       ]
     },
     "parsedSymbol": null,
@@ -23623,8 +23631,7 @@
       "parsedSymbol": "newton / meter",
       "commonCodeReferences": [
         "4P",
-        "M34",
-        "Q27"
+        "M34"
       ]
     },
     "parsedSymbol": "kilonewton / meter",
@@ -23641,8 +23648,7 @@
       "parsedSymbol": "newton / meter",
       "commonCodeReferences": [
         "4P",
-        "M34",
-        "Q27"
+        "M34"
       ]
     },
     "parsedSymbol": "poundal / inch",
@@ -23659,8 +23665,7 @@
       "parsedSymbol": "newton / meter",
       "commonCodeReferences": [
         "4P",
-        "M34",
-        "Q27"
+        "M34"
       ]
     },
     "parsedSymbol": "force_pound / yard",
@@ -27341,7 +27346,7 @@
         "M77"
       ]
     },
-    "parsedSymbol": "newton / meter",
+    "parsedSymbol": "newton",
     "categories": "LEVEL_3_INFORMATIVE"
   },
   {

diff --git a/python/tests/unece_excel_parser/test_unit.py b/python/tests/unece_excel_parser/test_unit.py
@@ -1,11 +1,11 @@
 import pytest
 from inflection import camelize
-
-from constants import *
 from lib.category import Category
 from lib.conversion_factor import ConversionFactor
 from lib.state import State
 
+from constants import *
+
 
 @pytest.mark.parametrize(
     "text_input",
@@ -75,3 +75,8 @@ def test_to_dict_contains_all_public_properties():
 def test_parse_unit_reference_does_parse_symbol_properly(text_input: str | None, expected_parsed_symbol: str | None):
     unit = Unit.parse(STATE, COMMON_CODE, NAME, DESCRIPTION, CATEGORY, text_input, CONVERSION_FACTOR)
     assert unit.parsed_symbol == expected_parsed_symbol
+
+
+def test_parse_unit_reference_does_replace_symbol_for_defined_common_codes():
+    unit = Unit.parse(STATE, "Q27", NAME, DESCRIPTION, CATEGORY, "N·m/m²", CONVERSION_FACTOR)
+    assert unit.parsed_symbol == "newton"