Merge branch 'master' into qc_mapping

.github/workflows/ci.yml

@@ -0,0 +1,73 @@
+name: GLAMOD marine-processing Testing Suite
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: py${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+        # Bookend python versions
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Fetch all history for all branches and tags.
+      - name: Set environment variables
+        run: |
+          echo "CONDA_ENV_FILE=ci/requirements/environment.yml" >> $GITHUB_ENV
+          echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
+
+      - name: Create conda environment
+        uses: mamba-org/provision-with-micromamba@main
+        with:
+          cache-downloads: true
+          cache-downloads-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+          micromamba-version: 'latest'
+          environment-file: ci/requirements/environment.yml
+          extra-specs: |
+            python=${{ matrix.python-version }}
+
+      - name: Install cdm reader mapper toolbox
+        run: |
+          python -m pip install --no-deps -e .
+
+      - name: Version info
+        run: |
+          conda info -a
+          conda list
+
+      - name: Import marine_processing toolbox
+        run: |
+          python -c "import glamod_marine_processing"
+
+      - name: Run tests
+        timeout-minutes: 10
+        run: python -u -m pytest
+          --cov=glamod_marine_processing
+          --cov-report=xml
+          --junitxml=test-results/${{ runner.os }}-${{ matrix.python-version }}.xml
+
+      - name: Upload code coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          flags: unittests
+          env_vars: RUNNER_OS,PYTHON_VERSION
+          name: codecov-umbrella
+          fail_ci_if_error: false

CHANGES.rst

@@ -18,10 +18,11 @@ New features and enhancements
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 * add some information files: ``AUTHORS.rst``, ``CHANGES.rst``, ``CONTRIBUTING.rst`` and ``LICENSE`` (:pull:`3`)
 * make us of pre-commit (:pull:`3`)
-* make use of an command-line interface to create suite PYTHON and SLURM scripts (:pull:`3`)
+* make use of an command-line interface to create suite PYTHON and SLURM scripts (:pull:`3`, :pull:`5`)
 * add new release 7.0 configuration files (:pull:`3`)
 * set some default directories and SLURM settings for both HPC systems KAY and MeluXina (:pull:`3`)
 
 Internal changes
 ^^^^^^^^^^^^^^^^
 * reduce complexity of some functions (:pull:`3`)
+* adding observational testing suite (:issue:`5`, :pull:`5`)

ci/requirements/environment.yml

@@ -0,0 +1,16 @@
+name: marine-processing-test
+
+channels:
+    - conda-forge
+    - nodefaults
+
+dependencies:
+    # for testing
+    - pytest
+    - pytest-cov
+    - pytest-env
+    - pip:
+      - git+https://github.com/glamod/cdm_reader_mapper
+      - pandas-validation
+      - Levenshtein
+      - simplejson

glamod_marine_processing/cli.py

@@ -107,6 +107,11 @@ def build_configuration(self):
         if self.work_directory is not None:
             config["paths"]["glamod"] = self.work_directory
 
+        try:
+            user = os.getlogin()
+        except OSError:
+            user = "testuser"
+
         config["paths"]["data_directory"] = get_abs_path(
             config["paths"]["data_directory"]
         )
@@ -121,7 +126,7 @@ def build_configuration(self):
         scripts_directory = os.path.join(code_directory, "scripts")
         lotus_scripts_directory = os.path.join(code_directory, "lotus_scripts")
         work_directory = os.path.abspath(config["paths"]["glamod"])
-        scratch_directory = os.path.join(work_directory, os.getlogin())
+        scratch_directory = os.path.join(work_directory, user)
         release_directory = os.path.join(
             scratch_directory, self.release, self.dataset, self.level
         )
@@ -155,7 +160,10 @@ def __init__(self):
             """,
         )
         self.submit_jobs = click.option(
-            "-submit", "--submit_jobs", is_flag=True, help="Submit job scripts"
+            "-submit", "--submit_jobs", is_flag=True, help="Submit job scripts."
+        )
+        self.run_jobs = click.option(
+            "-run", "--run_jobs", is_flag=True, help="Run job scripts interactively."
         )
         self.level = click.option(
             "-l",
@@ -248,7 +256,22 @@ def __init__(self):
             "-qc",
             "--quality_control",
             is_flag=True,
-            help="Do quality control for qc_suite only",
+            help="Do quality control for qc_suite only.",
+        )
+        self.source_pattern = click.option(
+            "-sp",
+            "--source_pattern",
+            help="User-defined input source pattern.",
+        )
+        self.prev_file_id = click.option(
+            "-p_id",
+            "--prev_file_id",
+            help="fileID of input file names. Default <YYYY><MM><RELEASE>",
+        )
+        self.external_qc_files = click.option(
+            "-ext_qc",
+            "--external_qc_files",
+            help="Path to external QC files. Default: <data_directory>/external_files.",
         )
 
 

glamod_marine_processing/cli_obs.py

@@ -24,10 +24,13 @@ def ObsCli(
     release,
     update,
     dataset,
+    source_pattern,
+    prev_file_id,
     data_directory,
     work_directory,
     config_file,
     submit_jobs,
+    run_jobs,
     overwrite,
 ):
     """Enry point for the obs_suite command line interface."""
@@ -64,8 +67,17 @@ def ObsCli(
 
     level_config = load_json(level_config_file)
     level_config["submit_jobs"] = submit_jobs
+    level_config["run_jobs"] = run_jobs
     level_config["level"] = level
     level_config["overwrite"] = overwrite
+    if source_pattern:
+        level_config["source_pattern"] = source_pattern
+    if prev_file_id:
+        if prev_file_id[0] != "*":
+            prev_file_id = f"*{prev_file_id}"
+        if prev_file_id[-1] != "*":
+            prev_file_id = f"{prev_file_id}*"
+        level_config["prev_fileID"] = prev_file_id
 
     for key, value in config.items():
         level_config[key] = value

glamod_marine_processing/cli_qc.py

@@ -26,11 +26,13 @@ def QcCli(
     corrections_version,
     data_directory,
     work_directory,
+    external_qc_files,
     config_file,
     submit_jobs,
     preprocessing,
     quality_control,
     high_resolution_qc,
+    run_jobs,
     overwrite,
 ):
     """Enry point for theqcmetadata_suite command line interface."""
@@ -58,8 +60,9 @@ def QcCli(
     )
     parameter_file = os.path.join(p.config_directory, "ParametersCCI.json")
     icoads_version = "3.0.2"
-    external_files = os.path.join(p.data_directory, "external_files")
-    sst_files = os.path.join(external_files, "SST")
+    if external_qc_files is None:
+        external_qc_files = os.path.join(p.data_directory, "external_files")
+    sst_files = os.path.join(external_qc_files, "SST")
     sst_stdev_climatology = os.path.join(sst_files, "OSTIA_pentad_stdev_climatology.nc")
     old_sst_stdev_climatology = os.path.join(
         sst_files, "HadSST2_pentad_stdev_climatology.nc"
@@ -73,12 +76,12 @@ def QcCli(
     sst_buddy_avg_sampling = os.path.join(
         sst_files, "OSTIA_buddy_range_sampling_error.nc"
     )
-    ostia_background = os.path.join(external_files, "OSTIA_background")
+    ostia_background = os.path.join(external_qc_files, "OSTIA_background")
     djf_ostia_background = os.path.join(ostia_background, "DJF_bckerr_smooth.nc")
     jja_ostia_background = os.path.join(ostia_background, "JJA_bckerr_smooth.nc")
     son_ostia_background = os.path.join(ostia_background, "SON_bckerr_smooth.nc")
     mam_ostia_background = os.path.join(ostia_background, "MAM_bckerr_smooth.nc")
-    test_files = os.path.join(external_files, "TestFiles")
+    test_files = os.path.join(external_qc_files, "TestFiles")
     sst_climatology_file = os.path.join(test_files, "HadSST2_pentad_climatology.nc")
     mat_climatology_file = os.path.join(test_files, "HadNMAT2_pentad_climatology.nc")
     stdev_climatology_file = os.path.join(
@@ -94,7 +97,7 @@ def QcCli(
         out_dir=out_dir,
         ICOADS_dir=icoads_dir,
         track_out_dir=out_dir,
-        external_files=external_files,
+        external_files=external_qc_files,
         key="Directories",
     )
 
@@ -135,6 +138,7 @@ def QcCli(
         key="TestFiles",
     )
     config["submit_jobs"] = submit_jobs
+    config["run_jobs"] = run_jobs
 
     mkdir(qc_log_directory)
     mkdir(qc_hr_log_directory)

glamod_marine_processing/obs_suite/configuration_files/release_7.0/000000/ICOADS_R3.0.2T/level2_cmd_add.json

@@ -1,67 +1,67 @@
 {
   "103-792": {
-    "year_init": 2015,
-    "year_end": 2021,
+    "year_init": 2022,
+    "year_end": 2022,
     "exclude": false,
     "params_exclude": []
   },
   "103-793": {
-    "year_init": 2015,
-    "year_end": 2021,
+    "year_init": 2022,
+    "year_end": 2022,
     "exclude": false,
     "params_exclude": []
   },
   "103-794": {
-    "year_init": 2015,
-    "year_end": 2021,
+    "year_init": 2022,
+    "year_end": 2022,
     "exclude": false,
     "params_exclude": []
   },
   "103-795": {
-    "year_init": 2015,
+    "year_init": 2022,
     "year_end": 2022,
     "exclude": false,
     "params_exclude": []
   },
   "103-797": {
-    "year_init": 2015,
-    "year_end": 2021,
+    "year_init": 2022,
+    "year_end": 2022,
     "exclude": false,
     "params_exclude": []
   },
   "114-992": {
-    "year_init": 2015,
-    "year_end": 2021,
+    "year_init": 2022,
+    "year_end": 2022,
     "exclude": false,
     "params_exclude": []
   },
   "114-993": {
-    "year_init": 2015,
-    "year_end": 2021,
+    "year_init": 2022,
+    "year_end": 2022,
     "exclude": false,
     "params_exclude": []
   },
   "114-994": {
-    "year_init": 2015,
-    "year_end": 2021,
+    "year_init": 2022,
+    "year_end": 2022,
     "exclude": false,
     "params_exclude": []
   },
   "114-995": {
-    "year_init": 2015,
-    "year_end": 2021,
+    "year_init": 2022,
+    "year_end": 2022,
     "exclude": false,
     "params_exclude": []
   },
   "172-798": {
-    "year_init": 2015,
-    "year_end": 2021,
+    "year_init": 2022,
+    "year_end": 2022,
     "exclude": false,
     "params_exclude": []
   },
   "params_exclude": [
     "observations-wbt"
   ],
-  "year_init": 2015,
-  "year_end": 2021
+  "year_init": 2022,
+  "year_end": 2022
 }

glamod_marine_processing/obs_suite/lotus_scripts/config_array.py

@@ -18,7 +18,13 @@
 
 # FUNCTIONS -------------------------------------------------------------------
 def config_element(
-    sid_dck_log_dir, ai, script_config, sid_dck, yyyy, mm, filename=None
+    sid_dck_log_dir,
+    ai,
+    script_config,
+    sid_dck,
+    yyyy,
+    mm,
+    filename=None,
 ):
     """Update configuration script."""
     script_config.update({"sid_dck": sid_dck})
@@ -60,10 +66,8 @@ def clean_ok_logs(
         for x in ok_files:
             os.remove(x)
     for source_file in source_files:
-        # print(source_file)
         yyyy, mm = get_yyyymm(source_file)
         if int(yyyy) >= year_init and int(yyyy) <= year_end:
-            # config_element(sid_dck_log_dir,ai,script_config,sid_dck,yyyy,mm, source_file)
             config_element(sid_dck_log_dir, ai, config, sid_dck, yyyy, mm, source_file)
             ai += 1
         elif (int(yyyy) == year_init - 1 and int(mm) == 12) or (
@@ -103,7 +107,6 @@ def clean_failed_logs_only(
         source_file = re.sub("[?]{2}", mm, source_file)
         source_file = os.path.join(source_dir, sid_dck, source_file)
         if int(yyyy) >= year_init and int(yyyy) <= year_end:
-            # config_element(sid_dck_log_dir,ai,script_config,sid_dck,yyyy,mm, source_file)
             config_element(sid_dck_log_dir, ai, config, sid_dck, yyyy, mm, source_file)
             ai += 1
         elif (int(yyyy) == year_init - 1 and int(mm) == 12) or (