Update data fetcher gdac/erddap to retrieve BGC variables (oxygen)

.github/workflows/pytests.yml

@@ -92,6 +92,7 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./cov.xml
+
           flags: unittests, core, pinned, py${{matrix.python-version}}
           name: codecov-github
           fail_ci_if_error: false

argopy/data_fetchers/erddap_data.py

@@ -48,6 +48,8 @@
     OPTIONS["erddap"] + "/info/ArgoFloats/index.json"
 )  # URL to check if the API is alive
 
+#class NewDataFetcher(ArgoDataFetcherProto):
+
 
 class ErddapArgoDataFetcher(ArgoDataFetcherProto):
     """Manage access to Argo data through Ifremer ERDDAP
@@ -79,6 +81,13 @@ def uri(self) -> list:
         """Return the list of Unique Resource Identifier (URI) to download data"""
         raise NotImplementedError("ErddapArgoDataFetcher.uri not implemented")
 
+    @staticmethod
+    def test_prise_en_compte_jup(self):
+        print('La modification a été prise en compte')
+
+    def variable_bgc(self):
+        print(self)
+
     ###
     # Methods that must not change
     ###
@@ -250,6 +259,13 @@ def _add_attributes(self, this):  # noqa: C901
                 "convention": "R : real time; D : delayed mode; A : real time with adjustment",
             }
 
+        # BGC variables
+        if "PARAMAMETER_DATA_MODE" in this.data_vars:
+            this["PARAMETER_DATA_MODE"].attrs = {
+                "long_name": "Delayed mode or real time data for BGC variables",
+                "convention": "R : real time; D : delayed mode; A : real time with adjustment",
+            }
+
         if "DIRECTION" in this.data_vars:
             this["DIRECTION"].attrs = {
                 "long_name": "Direction of the station profiles",
@@ -326,7 +342,101 @@ def _minimal_vlist(self):
                 "config_mission_number",
             ]
             [vlist.append(p) for p in plist]
-
+            # TODO : implement two modes (standard/expert) for two lists of possible BGC variables
+
+            if OPTIONS.get('mode')  == "standard":
+                plist = [
+                    "pres",
+                    "temp",
+                    "psal",
+                    "doxy",
+                ]
+
+            elif OPTIONS.get('mode')  == "expert":
+                plist = [
+                    "pres",
+                    "temp",
+                    "psal",
+#                    "cndc",
+                    "doxy",
+#                    "beta_backscattering",
+#                    "fluorescence_chla",
+#                    "fluorescence_cdom",
+#                    "side_scattering_turbidity", #1
+#                    "transmittance_particle_beam_attenuation", #1
+#                    "bbp",
+#                    "turbidity",
+#                    "cp",
+#                    "chla",
+#                    "cdom",
+#                    "nitrate",
+#                    "temp_doxy",
+#                    "temp_voltage_doxy",
+#                    "voltage_doxy",
+#                    "frequency_doxy",
+#                    "count_doxy",
+#                    "bphase_doxy",
+#                    "dphase_doxy",
+#                    "tphase_doxy",
+#                    "c1phase_doxy",
+#                    "c2phase_doxy",
+#                    "molar_doxy",
+#                    "phase_delay_doxy",
+#                    "mlpl_doxy",
+#                    "nb_sample",
+#                    "rphase_doxy",
+#                    "temp_count",
+#                    "led_flashing_count",
+#                    "ppox_doxy",
+#                    "beta_backscattering470",
+#                    "beta_backscattering532",
+#                    "beta_backscattering700",
+#                    "temp_cpu_chla",
+#                    "bbp470",
+#                    "bbp532",
+#                    "bbp700",
+#                    "uv_intensity_nitrate",
+#                    "uv_intensity_dark_nitrate",
+#                    "uv_intensity_dark_sea_water",
+#                    "bisulfide",
+#                    "molar_nitrate",
+#                    "fit_error",
+#                    "temp_nitrate",
+#                    "temp_spectrophotometer_nitrate",
+#                    "humidity_nitrate",
+#                    "vrs_pH",
+#                    "temp_ph",
+#                    "ib_ph",
+#                    "vk_ph",
+#                    "ik_ph",
+#                    "ph_in_situ_total",
+#                    "ph_in_situ_free",
+#                    "raw_downwelling_irradiance",
+#                    "raw_downwelling_irradiance380",
+#                    "raw_downwelling_irradiance412",
+#                    "raw_downwelling_irradiance490",
+#                    "raw_downwelling_irradiance555",
+#                    "down_irradiance",
+#                    "down_irradiance380",
+#                    "down_irradiance412",
+#                    "down_irradiance490",
+#                    "down_irradiance555",
+#                    "raw_upwelling_radiance",
+#                    "raw_upwelling_radiance412",
+#                    "raw_upwelling_radiance443",
+#                    "raw_upwelling_radiance490",
+#                    "raw_upwelling_radiance555",
+#                    "up_radiance",
+#                    "up_radiance412",
+#                    "up_radiance443",
+#                    "up_radiance490",
+#                    "up_radiance555",
+#                    "raw_downwelling_par",
+#                    "downwelling_par",
+#                    "tilt",
+#                    "mtime",
+                ]
+
             plist = [
                 "pres",
                 "temp",
@@ -359,6 +469,54 @@ def _minimal_vlist(self):
 
         return vlist
 
+    @property
+    def _dtype(self):
+        """ Return a dictionary of data types for each variable requested to erddap in the minimal vlist """
+        # not up to date : TODO
+        dref = {
+            "data_mode": object,
+            "latitude": np.float64,
+            "longitude": np.float64,
+            "position_qc": np.int64,
+            "time": object,
+            "time_qc": np.int64,
+            "direction": object,
+            "platform_number": np.int64,
+            "config_mission_number": np.int64,
+            "vertical_sampling_scheme": object,
+            "cycle_number": np.int64,
+            "pres": np.float64,
+            "temp": np.float64,
+            "psal": np.float64,
+            "doxy": np.float64,
+            "pres_qc": np.int64,
+            "temp_qc": object,
+            "psal_qc": object,
+            "doxy_qc": object,
+            "pres_adjusted": np.float64,
+            "temp_adjusted": np.float64,
+            "psal_adjusted": np.float64,
+            "doxy_adjusted": np.float64,
+            "pres_adjusted_qc": object,
+            "temp_adjusted_qc": object,
+            "psal_adjusted_qc": object,
+            "doxy_adjusted_qc": object,
+            "pres_adjusted_error": np.float64,
+            "temp_adjusted_error": np.float64,
+            "psal_adjusted_error": np.float64,
+            "doxy_adjusted_error": np.float64,
+            "ptmp": np.float64,
+        }
+        plist = self._minimal_vlist
+        response = {}
+        for p in plist:
+            if p in dref:
+                response[p] = dref[p]
+            else:
+                response[p] = object
+        return response
+
+
     def cname(self):
         """Return a unique string defining the constraints"""
         return self._cname()
@@ -555,7 +713,6 @@ def init(self, WMO=[], CYC=None, **kw):
             self.definition = "%s for profiles" % self.definition
         else:
             self.definition = "%s for floats" % self.definition
-
         return self
 
     def define_constraints(self):
@@ -624,6 +781,10 @@ def init(self, box: list, **kw):
             self.definition = (
                 "Ifremer erddap Argo REFERENCE data fetcher for a space/time region"
             )
+        elif self.dataset_id == "bgc":
+            self.definition = (
+                "Ifremer erddap Argo bgc data fetcher for a space/time region"
+            )
 
         return self
 
@@ -663,3 +824,4 @@ def uri(self):
                     ).get_url()
                 )
             return urls
+

argopy/data_fetchers/gdacftp_data.py

@@ -345,6 +345,8 @@ def to_xarray(self, errors: str = "ignore"):
             decode_cf=1,
             use_cftime=0,
             mask_and_scale=1,
+            #engine= #TODO
+            #self.
         )
 
         # Data post-processing:

argopy/data_fetchers/proto.py

@@ -102,6 +102,10 @@ def sha(self) -> str:
 
     def dashboard(self, **kw):
         """Return 3rd party dashboard for the access point"""
+        if 'type' not in kw:
+            if self.dataset_id == 'bgc':
+                kw['type'] = 'bgc'
+
         if self.WMO is not None:
             if len(self.WMO) == 1 and self.CYC is not None and len(self.CYC) == 1:
                 return dashboard(wmo=self.WMO[0], cyc=self.CYC[0], **kw)

argopy/plot/argo_colors.py

@@ -86,7 +86,7 @@ def __init__(self, name: str = "Set1", N: int = None):
                 "ticklabels": ['PROBABLE', 'CONFIRMED', 'REGISTERED', 'OPERATIONAL', 'INACTIVE', 'CLOSED'],
             },
             "qc": {
-                "name": "Quality control flag scale",
+                "name": "Quality control flag scale (RR2, RD2)",
                 "aka": ["qc_flag", "quality_control", "quality_control_flag", "quality_control_flag_scale"],
                 "constructor": self._colormap_quality_control_flag,
                 "ticks": np.arange(0, 9 + 1),
@@ -109,6 +109,20 @@ def __init__(self, name: str = "Set1", N: int = None):
                 "ticklabels": ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
                                "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"],
             },
+            "prof_qc": {
+                "name": "Profile quality control flag scale (RP2)",
+                "aka": ["prof_qc_flag", "prof_quality_control"],
+                "constructor": self._colormap_profile_quality_control_flag,
+                "ticks": ["A", "B", "C", "D", "E", "F", "N", " "],
+                "ticklabels": [ "100% of profile good",
+                                "75% - 100% of profile good",
+                                "50% - 75% of profile good",
+                                "25% - 50% of profile good",
+                                "0% - 25% of profile good",
+                                "0% of profile good",
+                                "No QC",
+                                "FillValue"],
+            },
         }
         self.registered = self.name in self.list_valid_known_colormaps
         self._colormap = self.cmap
@@ -274,6 +288,20 @@ def _colormap_quality_control_flag(self):
                  ]
         return mcolors.LinearSegmentedColormap.from_list(self.definition['name'], clist, 10)
 
+    def _colormap_profile_quality_control_flag(self):
+        """Return a colormap for profile QC flag"""
+        clist = ['#00DF4E',
+                 '#1DFF88',
+                 '#D8FE00',
+                 '#F9FE00',
+                 '#FF9900',
+                 '#FE0000',
+                 '#000000',
+                 '#FF42A1'
+                 ]
+        return mcolors.LinearSegmentedColormap.from_list(self.definition['name'], clist, 8)
+
+
     @property
     def cmap(self):
         """Discrete colormap as :class:`matplotlib.colors.LinearSegmentedColormap`

argopy/plot/plot.py

@@ -496,12 +496,16 @@ def guess_yvar(data):
         }
         if isinstance(data, pd.DataFrame) and not legend:
             scatter_opts['legend'] = False  # otherwise Pandas will add a legend even if we set legend=False
-        sc = group.plot.scatter(
-            x=x, y=y,
-            ax=ax,
-            **scatter_opts
-        )
-        patches.append(sc)
+        try:
+            sc = group.plot.scatter(
+                x=x, y=y,
+                ax=ax,
+                **scatter_opts
+            )
+            patches.append(sc)
+        except:
+            log.debug("Error with scatter plot options: %s" % str(scatter_opts))
+            raise
 
     if cbar:
         if cbarlabels == 'auto':