Feature 282 enhance schema

METdbLoad/sql/mv_mysql.sql

@@ -458,6 +458,8 @@ CREATE TABLE line_data_ecnt
     me_ge_obs        DOUBLE,
     n_lt_obs         INT,
     me_lt_obs        DOUBLE,
+    ign_conv_oerr    DOUBLE,
+    ign_corr_oerr    DOUBLE,
 
     CONSTRAINT line_data_ecnt_data_file_id_pk
         FOREIGN KEY (data_file_id)
@@ -910,6 +912,9 @@ CREATE TABLE line_data_vl1l2
     uvoobar        DOUBLE,
     f_speed_bar    DOUBLE DEFAULT -9999,
     o_speed_bar    DOUBLE DEFAULT -9999,
+    dir_me         DOUBLE,
+    dir_mae        DOUBLE,
+    dir_mse        DOUBLE,
 
     CONSTRAINT line_data_vl1l2_data_file_id_pk
         FOREIGN KEY (data_file_id)
@@ -948,6 +953,9 @@ CREATE TABLE line_data_val1l2
     uvooabar       DOUBLE,
     fa_speed_bar   DOUBLE DEFAULT -9999,
     oa_speed_bar   DOUBLE DEFAULT -9999,
+    dira_me        DOUBLE,
+    dira_mae       DOUBLE,
+    dira_mse       DOUBLE,
 
     CONSTRAINT line_data_val1l2_data_file_id_pk
         FOREIGN KEY (data_file_id)
@@ -1706,6 +1714,19 @@ CREATE TABLE line_data_vcnt
     anom_corr_uncntr     DOUBLE DEFAULT -9999,
     anom_corr_uncntr_bcl DOUBLE DEFAULT -9999,
     anom_corr_uncntr_bcu DOUBLE DEFAULT -9999,
+    dir_me,              DOUBLE,
+    dir_me_bcl           DOUBLE,
+    dir_me_bcu           DOUBLE,
+    dir_mae              DOUBLE,
+    dir_mae_bcl          DOUBLE,
+    dir_mae_bcu          DOUBLE,
+    dir_mse              DOUBLE,
+    dir_mse_bcl          DOUBLE,
+    dir_mse_bcu          DOUBLE,
+    dir_rmse             DOUBLE,
+    dir_rmse_bcl         DOUBLE,
+    dir_rmse_bcu         DOUBLE,
+
 
     CONSTRAINT line_data_vcnt_data_file_id_pk
         FOREIGN KEY (data_file_id)

METdbLoad/sql/updates/update_for_5_1_beta4.sql

@@ -0,0 +1,50 @@
+DELIMITER |
+
+
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_me DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_me_bcl DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_me_bcu DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_mae DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_mae_bcl DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_mae_bcu DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_mse DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_mse_bcl DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_mse_bcu DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_rmse DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_rmse_bcl DOUBLE |
+ALTER TABLE line_data_vcnt
+    ADD COLUMN dir_rmse_bcu DOUBLE |
+
+ALTER TABLE line_data_val1l2
+    ADD COLUMN dira_me DOUBLE |
+ALTER TABLE line_data_val1l2
+    ADD COLUMN dira_mae DOUBLE |
+ALTER TABLE line_data_val1l2
+    ADD COLUMN dira_mse DOUBLE |
+
+ALTER TABLE line_data_vl1l2
+    ADD COLUMN dir_me DOUBLE |
+ALTER TABLE line_data_vl1l2
+    ADD COLUMN dir_mae DOUBLE |
+ALTER TABLE line_data_vl1l2
+    ADD COLUMN dir_mse DOUBLE |
+
+ALTER TABLE line_data_ecnt
+    ADD COLUMN ign_conv_oerr DOUBLE |
+ALTER TABLE  line_data_ecnt
+    ADD COLUMN ign_corr_oerr DOUBLE |
+
+
+
+DELIMITER ;

METdbLoad/tests/update_schema_6.0_beta4/test_loading.py

@@ -0,0 +1,271 @@
+import pytest
+import pymysql
+import yaml
+from dataclasses import make_dataclass
+
+#######################################################################
+# These tests can only be run on the host where the database is running.
+# Pre-condition:
+#     The data in the accompanying data directory ./Data, should
+#     already be loaded in the database using the corresponding
+#     schema: mv_mysql.sql and the appropriate xml specification file
+#
+
+CONST_LOAD_DB_CMD = "use mv_load_test"
+
+@pytest.fixture
+def setup_db():
+
+    """
+          Read in the config file to retrieve the database login information.
+
+    """
+    config_file = 'test_loading.yaml'
+    with open(config_file, 'r') as stream:
+        try:
+            parms: dict = yaml.load(stream, Loader=yaml.FullLoader)
+            # pathlib.Path(parms['output_dir']).mkdir(parents=True, exist_ok=True)
+        except yaml.YAMLError as exc:
+            print(exc)
+
+    # Create a dataclass of the database information
+    #TCDiag = make_dataclass("TCDiag", ["total", "index", "diag_src", "diag_val"], frozen=True)
+    #orig = TCDiag(orig_total, orig_index, orig_diag_src, orig_diag_val)
+    DBS = make_dataclass("DBS", ["hostname", "username", "password", "dbname"])
+    db_settings = DBS(parms['hostname'], parms['username'], parms['password'], parms['dbname'])
+
+    # Return the db settings (hostname, username, etc.)
+    yield db_settings
+
+
+def test_ecnt_db_created(setup_db):
+
+    # log into the database and verify the database exists, tables exist, new columns for each affected
+    # table exists, and check that for specific ign_conv_oerr and ign_corr_oer values, only one row is
+    # found.
+
+
+    conn = pymysql.connect(
+        host=setup_db.hostname,
+        user=setup_db.username,
+        password=setup_db.password,
+        db=setup_db.dbname,
+        charset='utf8mb4'
+    )
+
+    try:
+        with conn.cursor() as cursor:
+            # Check that the mv_load_test database was created
+            check_db_exists_query = "show databases;"
+            cursor.execute(check_db_exists_query)
+
+            # Get all rows
+            rows = cursor.fetchall()
+            list_of_rows = [r[0] for r in rows]
+
+            #Results
+            assert 'mv_load_test' in list_of_rows
+
+
+
+    finally:
+        conn.close()
+
+def test_tables_created(setup_db):
+
+    # log into the database and verify the ECNT, VCNT, VL1L2, and VAL1L2 tables exist
+
+
+    conn = pymysql.connect(
+        host=setup_db.hostname,
+        user=setup_db.username,
+        password=setup_db.password,
+        db=setup_db.dbname,
+        charset='utf8mb4'
+    )
+
+    try:
+        with conn.cursor() as cursor:
+            # Check that the line_data_ecnt, line_data_vcnt, line_data_vl1l2, and
+            # line_data_val1l2 tables were created
+            cursor.execute(CONST_LOAD_DB_CMD)
+
+            check_tables_exist = "show tables;"
+            cursor.execute(check_tables_exist)
+
+            # Get all rows
+            rows = cursor.fetchall()
+            list_of_rows = [r[0] for r in rows]
+            assert 'line_data_ecnt' in list_of_rows
+            assert 'line_data_vcnt' in list_of_rows
+            assert 'line_data_vl1l2' in list_of_rows
+            assert 'line_data_val1l2' in list_of_rows
+
+    finally:
+        conn.close()
+
+
+def test_ecnt_columns(setup_db):
+   # log into the database and verify the ign_conv_oerr and ign_corr_oerr columns are in the
+   # list_data_ecnt database table.
+
+    conn = pymysql.connect(
+        host=setup_db.hostname,
+        user=setup_db.username,
+        password=setup_db.password,
+        db=setup_db.dbname,
+        charset='utf8mb4'
+    )
+
+    try:
+        with conn.cursor() as cursor:
+            # Check that the line_data_ecnt, line_data_vcnt, line_data_vl1l2, and
+            # line_data_val1l2 tables were created
+            cursor.execute(CONST_LOAD_DB_CMD)
+
+
+            check_columns_exist = "desc line_data_ecnt;"
+            cursor.execute(check_columns_exist)
+
+            # Get all rows
+            rows = cursor.fetchall()
+            list_of_rows = [r[0] for r in rows]
+            assert 'ign_conv_oerr' in list_of_rows
+            assert 'ign_corr_oerr' in list_of_rows
+
+    finally:
+        conn.close()
+
+def test_vcnt_columns(setup_db):
+   # log into the database and verify the dir_me, dir_me_bcl, dir_me_bcu, ..., etc. columns are in the
+   # list_data_ecnt database table.
+
+    expected_cols = ['dir_me', 'dir_me_bcl', 'dir_me_bcu',
+                     'dir_mae', 'dir_mae_bcl', 'dir_mae_bcu',
+                     'dir_mse', 'dir_mse_bcl', 'dir_mse_bcu',
+                     'dir_rmse', 'dir_rmse_bcl', 'dir_rmse_bcu'
+                     ]
+    conn = pymysql.connect(
+        host=setup_db.hostname,
+        user=setup_db.username,
+        password=setup_db.password,
+        db=setup_db.dbname,
+        charset='utf8mb4'
+    )
+
+    try:
+        with conn.cursor() as cursor:
+            # Check that the  line_data_vcnt expected columns were created
+            cursor.execute(CONST_LOAD_DB_CMD)
+
+            check_columns_exist = "desc line_data_vcnt;"
+            cursor.execute(check_columns_exist)
+
+            # Get all rows
+            rows = cursor.fetchall()
+            list_of_rows = [r[0] for r in rows]
+            for expected in expected_cols:
+                assert expected in list_of_rows
+                assert expected in list_of_rows
+
+    finally:
+        conn.close()
+
+def test_vl1l2_columns(setup_db):
+   # log into the database and verify the dir_me, dir_mae, and dir_mse columns are in the
+   # list_data_vl1l2 database table.
+
+    expected_cols = ['dir_me', 'dir_mae', 'dir_mse']
+    conn = pymysql.connect(
+        host=setup_db.hostname,
+        user=setup_db.username,
+        password=setup_db.password,
+        db=setup_db.dbname,
+        charset='utf8mb4'
+    )
+
+    try:
+        with conn.cursor() as cursor:
+            # Check that the  line_data_vl1l2 table has the expected columns
+            cursor.execute(CONST_LOAD_DB_CMD)
+
+            check_columns_exist = "desc line_data_vl1l2;"
+            cursor.execute(check_columns_exist)
+
+            # Get all rows
+            rows = cursor.fetchall()
+            list_of_rows = [r[0] for r in rows]
+            for expected in expected_cols:
+                assert expected in list_of_rows
+                assert expected in list_of_rows
+
+    finally:
+        conn.close()
+
+def test_val1l2_columns(setup_db):
+   # log into the database and verify the dira_me, dira_mae, and dira_mse columns are in the
+   # list_data_val1l2 database table.
+
+    expected_cols = ['dira_me', 'dira_mae', 'dira_mse']
+    conn = pymysql.connect(
+        host=setup_db.hostname,
+        user=setup_db.username,
+        password=setup_db.password,
+        db=setup_db.dbname,
+        charset='utf8mb4'
+    )
+
+    try:
+        with conn.cursor() as cursor:
+            # Check that the line_data_vl1l2 table has the expected columns
+            cursor.execute(CONST_LOAD_DB_CMD)
+
+            check_columns_exist = "desc line_data_val1l2;"
+            cursor.execute(check_columns_exist)
+
+            # Get all rows
+            rows = cursor.fetchall()
+            list_of_rows = [r[0] for r in rows]
+            for expected in expected_cols:
+                assert expected in list_of_rows
+                assert expected in list_of_rows
+
+    finally:
+        conn.close()
+
+
+def test_ecnt_vals(setup_db):
+   # log into the database and verify the ECNT values for ign_conv_oerr and
+   # ign_corr_oerr result in a result. This verifies that the input data
+   # was correctly loaded for the ECNT data.
+
+    ign_conv_oerr = "33.41424"
+    ign_corr_oerr = "440.06905"
+
+    conn = pymysql.connect(
+        host=setup_db.hostname,
+        user=setup_db.username,
+        password=setup_db.password,
+        db=setup_db.dbname,
+        charset='utf8mb4'
+    )
+
+    try:
+        with conn.cursor() as cursor:
+            # Check that the line_data_vl1l2 table has the expected columns
+            cursor.execute(CONST_LOAD_DB_CMD)
+
+            check_columns_exist = "select * from line_data_ecnt where ign_conv_oerr = " + ign_conv_oerr + \
+                                  " AND ign_corr_oerr = " + ign_corr_oerr +  ";"
+            cursor.execute(check_columns_exist)
+
+            # Get all rows
+            rows = cursor.fetchall()
+            # Only one row should correspond to this query
+            assert len(rows) == 1
+
+
+    finally:
+        conn.close()
+
+

METdbLoad/tests/update_schema_6.0_beta4/test_loading.yaml

@@ -0,0 +1,6 @@
+hostname: 'localhost:3306'
+username: 'mvadmin'
+password: '160GiltVa0D5M'
+dbname: 'mv_load_test'
+output_dir: './output'
+