Update metview and setup-tools , python version.

Dockerfile

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG py_version=3.9
+ARG py_version=3.8
 FROM apache/beam_python${py_version}_sdk:2.40.0 as beam_sdk
 FROM continuumio/miniconda3:latest
 

environment.yml

@@ -15,8 +15,9 @@ name: era5
 channels:
   - conda-forge
 dependencies:
-  - python=3.9.17
+  - python=3.8.19
   - metview-batch==5.17.0
   - pip:
     - cython==0.29.34
+    - setuptools==70.3.0
     - metview

src/arco_era5/ingest_data_in_zarr.py

@@ -90,7 +90,6 @@ def ingest_data_in_zarr_dataflow_job(target_path: str, region: str, start_date:
             f"--temp_location gs://{bucket}/temp --runner DataflowRunner "
             f"--project {project} --region {region} --experiments use_runner_v2 "
             f"--worker_machine_type n2-highmem-8 --disk_size_gb 250 "
-            f"--setup_file /arco-era5/setup.py "
             f"--job_name {job_name} --number_of_worker_harness_threads 1 "
             f"--sdk_container_image {sdk_container_image} "
             f"--init_date {init_date}")

src/arco_era5/update_model_level_native_vertical_zarr.py

@@ -37,14 +37,6 @@
 from .constant import variables_full_names, zarr_files
 logger = logging.getLogger(__name__)
 
-try:
-    import metview as mv
-    Fieldset = mv.bindings.Fieldset
-except (ModuleNotFoundError, ImportError, FileNotFoundError):
-    logger.error('Metview could not be imported.')
-    mv = None  # noqa
-    Fieldset = Any
-
 TIME_RESOLUTION_HOURS = 1
 HOURS_PER_DAY = 24
 
@@ -88,6 +80,11 @@ def process_hourly_data(self, data_list: List[Any]) -> xr.Dataset:
         Returns:
             xr.Dataset: The processed dataset.
         """
+        try:
+            import metview as mv
+        except (ModuleNotFoundError, ImportError, FileNotFoundError):
+            logger.error('Metview could not be imported.')
+            mv = None  # noqa
         try:
             wind_fieldset, moisture_fieldset, surface_fieldset = data_list
 
@@ -144,6 +141,12 @@ def process(self, args: Tuple[int, int, int, int]):
         moisture_slice = ml_moisture.sel(time=current_timestamp).compute()
         surface_slice = sl_surface.sel(time=current_timestamp).compute()
 
+        try:
+            import metview as mv
+        except (ModuleNotFoundError, ImportError, FileNotFoundError):
+            logger.error('Metview could not be imported.')
+            mv = None  # noqa
+
         wind_fieldset = mv.dataset_to_fieldset(self.attribute_fix(wind_slice).squeeze())
         moisture_fieldset = mv.dataset_to_fieldset(self.attribute_fix(moisture_slice).squeeze())
         surface_fieldset = mv.dataset_to_fieldset(self.attribute_fix(surface_slice).squeeze())