Merge pull request #383 from CCI-Tools/381-nf-op_deprecation

resolves #381

cate/cli/main.py

@@ -343,8 +343,12 @@ def _get_op_io_info_str(inputs_or_outputs: dict, title_singular: str, title_plur
     op_info_str = ''
     op_info_str += '\n'
     if inputs_or_outputs:
+        inputs_or_outputs = {name: properties for name, properties in inputs_or_outputs.items()
+                             if not properties.get('deprecated')}
         op_info_str += '%s:' % (title_singular if len(inputs_or_outputs) == 1 else title_plural)
         for name, properties in inputs_or_outputs.items():
+            if properties.get('deprecated'):
+                continue
             op_info_str += '\n'
             op_info_str += '  %s (%s)' % (name, _get_op_data_type_str(properties.get('data_type', object)))
             description = properties.get('description', None)
@@ -1029,6 +1033,8 @@ def configure_parser_and_subparsers(cls, parser, subparsers):
                                  help="List only operations tagged by TAG or "
                                       "that have TAG in one of their tags. "
                                       "The comparison is case insensitive.")
+        list_parser.add_argument('--deprecated', '-d', action='store_true',
+                                 help="List deprecated operations.")
         list_parser.add_argument('--internal', '-i', action='store_true',
                                  help='List operations tagged "internal".')
         list_parser.set_defaults(sub_command_function=cls._execute_list)
@@ -1042,11 +1048,13 @@ def configure_parser_and_subparsers(cls, parser, subparsers):
     def _execute_list(cls, command_args):
         op_regs = OP_REGISTRY.op_registrations
 
-        def _is_op_selected(op_reg, tag_part: str, is_internal: bool):
+        def _is_op_selected(op_reg, tag_part: str, internal_only: bool, deprecated_only: bool):
+            if deprecated_only and not op_reg.op_meta_info.header.get('deprecated'):
+                return False
             tags = to_list(op_reg.op_meta_info.header.get('tags'))
             if tags:
                 # Tagged operations
-                if is_internal:
+                if internal_only:
                     if 'internal' not in tags:
                         return False
                 else:
@@ -1058,13 +1066,13 @@ def _is_op_selected(op_reg, tag_part: str, is_internal: bool):
                         return any(tag_part in tag.lower() for tag in tags)
                     elif isinstance(tags, str):
                         return tag_part in tags.lower()
-            elif is_internal or tag_part:
+            elif internal_only or tag_part:
                 # Untagged operations
                 return False
             return True
 
         op_names = sorted([op_name for op_name, op_reg in op_regs.items() if
-                           _is_op_selected(op_reg, command_args.tag, command_args.internal)])
+                           _is_op_selected(op_reg, command_args.tag, command_args.internal, command_args.deprecated)])
         name_pattern = None
         if command_args.name:
             name_pattern = command_args.name

cate/core/op.py

@@ -402,7 +402,11 @@ def __repr__(self):
 OP_REGISTRY = _DefaultOpRegistry()
 
 
-def op(registry=OP_REGISTRY, **properties):
+def op(tags=UNDEFINED,
+       version=UNDEFINED,
+       deprecated=UNDEFINED,
+       registry=OP_REGISTRY,
+       **properties):
     """
     ``op`` is a decorator function that registers a Python function or class in the default operation registry or
     the one given by *registry*, if any.
@@ -418,13 +422,21 @@ def op(registry=OP_REGISTRY, **properties):
 
     @op(version='X.x')
 
-    :param properties: Other properties (keyword arguments) that will be added to the meta-information of operation.
+    :param tags: An optional list of string tags.
+    :param version: An optional version string.
+    :param deprecated: An optional boolean. If set to ``True``, the operation's doc-string should explain why it
+           has been deprecated and which operation to use instead.
     :param registry: The operation registry.
+    :param properties: Other properties (keyword arguments) that will be added to the meta-information of operation.
     """
 
     def decorator(op_func):
+        new_properties = dict(tags=tags,
+                              version=version,
+                              deprecated=deprecated,
+                              **properties)
         op_registration = registry.add_op(op_func, fail_if_exists=False)
-        op_registration.op_meta_info.header.update({k: v for k, v in properties.items() if v is not UNDEFINED})
+        op_registration.op_meta_info.header.update({k: v for k, v in new_properties.items() if v is not UNDEFINED})
         return op_registration
 
     return decorator
@@ -438,6 +450,7 @@ def op_input(input_name: str,
              value_set_source=UNDEFINED,
              value_set=UNDEFINED,
              value_range=UNDEFINED,
+             deprecated=UNDEFINED,
              position=UNDEFINED,
              context=UNDEFINED,
              registry=OP_REGISTRY,
@@ -476,6 +489,8 @@ def op_input(input_name: str,
     :param value_set: A sequence of the valid values. Note that all values in this sequence
            must be compatible with *data_type*.
     :param value_range: A sequence specifying the possible range of valid values.
+    :param deprecated: An optional boolean. If set to ``True``, the input's doc-string should explain why the input
+           has been deprecated and which new input to use instead.
     :param position: The zero-based position of an input.
     :param context: If ``True``, the value of the operation input will be a dictionary representing
            the current execution context. For example,
@@ -502,6 +517,7 @@ def decorator(op_func):
                               value_set_source=value_set_source,
                               value_set=value_set,
                               value_range=value_range,
+                              deprecated=deprecated,
                               position=position,
                               context=context,
                               **properties)

cate/ops/io.py

@@ -38,14 +38,16 @@
 
 
 @op(tags=['input'])
-@op_input('ds_name')
+@op_input('ds_id')
+@op_input('ds_name', deprecated=True)
 @op_input('time_range', data_type=TimeRangeLike)
 @op_input('region', data_type=PolygonLike)
 @op_input('var_names', data_type=VarNamesLike)
 @op_input('normalize')
 @op_input('force_local')
 @op_input('local_ds_id')
 def open_dataset(ds_name: str,
+                 ds_id: str = None,
                  time_range: TimeRangeLike.TYPE = None,
                  region: PolygonLike.TYPE = None,
                  var_names: VarNamesLike.TYPE = None,
@@ -56,7 +58,8 @@ def open_dataset(ds_name: str,
     """
     Open a dataset from a data source identified by *ds_name*.
 
-    :param ds_name: The name of data source.
+    :param ds_name: The name of data source. This parameter has been deprecated, please use *ds_id* instead.
+    :param ds_id: The identifier for the data source.
     :param time_range: Optional time range of the requested dataset
     :param region: Optional spatial region of the requested dataset
     :param var_names: Optional names of variables of the requested dataset
@@ -67,9 +70,12 @@ def open_dataset(ds_name: str,
     :return: An new dataset instance.
     """
     import cate.core.ds
-    ds = cate.core.ds.open_dataset(data_source=ds_name, time_range=time_range,
-                                   var_names=var_names, region=region,
-                                   force_local=force_local, local_ds_id=local_ds_id,
+    ds = cate.core.ds.open_dataset(data_source=ds_id or ds_name,
+                                   time_range=time_range,
+                                   var_names=var_names,
+                                   region=region,
+                                   force_local=force_local,
+                                   local_ds_id=local_ds_id,
                                    monitor=monitor)
     if ds and normalize:
         return normalize_op(ds)

cate/webapi/websocket.py

@@ -232,18 +232,23 @@ def remove_local_datasource(self, data_source_name: str, remove_files: bool) ->
         data_store.remove_data_source(data_source_name, remove_files)
         return self.get_data_sources('local', monitor=Monitor.NONE)
 
-    def get_operations(self) -> List[dict]:
+    def get_operations(self, registry=None) -> List[dict]:
         """
         Get registered operations.
 
         :return: JSON-serializable list of data sources, sorted by name.
         """
+        registry = registry or OP_REGISTRY
         op_list = []
-        for op_name, op_reg in OP_REGISTRY.op_registrations.items():
+        for op_name, op_reg in registry.op_registrations.items():
+            if op_reg.op_meta_info.header.get('deprecated'):
+                continue
             op_json_dict = op_reg.op_meta_info.to_json_dict()
             op_json_dict['name'] = op_name
-            op_json_dict['inputs'] = [dict(name=name, **props) for name, props in op_json_dict['inputs'].items()]
-            op_json_dict['outputs'] = [dict(name=name, **props) for name, props in op_json_dict['outputs'].items()]
+            op_json_dict['inputs'] = [dict(name=name, **props) for name, props in op_json_dict['inputs'].items()
+                                      if not props.get('deprecated')]
+            op_json_dict['outputs'] = [dict(name=name, **props) for name, props in op_json_dict['outputs'].items()
+                                       if not props.get('deprecated')]
             op_list.append(op_json_dict)
 
         return sorted(op_list, key=lambda op: op['name'])

scripts/support/openall.py

@@ -23,7 +23,7 @@ def test_data_source(data_source, out_dir, result):
     dataset = None
     t0 = time.clock()
     try:
-        dataset = open_dataset(data_source_id,
+        dataset = open_dataset(ds_id=data_source_id,
                                force_local=True,
                                monitor=ConsoleMonitor(stay_in_line=True,
                                                       progress_bar_size=60))

test/cli/test_cli.py

@@ -285,6 +285,7 @@ def test_op_list(self):
         self.assert_main(['op', 'list', '--internal'], expected_stdout=['4 operations found'])
         self.assert_main(['op', 'list', '--tag', 'input'], expected_stdout=['8 operations found'])
         self.assert_main(['op', 'list', '--tag', 'output'], expected_stdout=['6 operations found'])
+        self.assert_main(['op', 'list', '--deprecated'], expected_stdout=['No operations found'])
 
 
 @unittest.skip(reason='Hardcoded values from remote service, contains outdated assumptions')

test/core/test_wsmanag.py

@@ -185,7 +185,7 @@ def test_session(self):
 
         self.del_base_dir(base_dir)
 
-    def test_persitence(self):
+    def test_persistence(self):
         base_dir = self.new_base_dir('TESTOMAT')
 
         workspace_manager = self.new_workspace_manager()

test/ops/test_io.py

@@ -14,23 +14,23 @@ class TestIO(TestCase):
     @unittest.skip(reason="skipped unless you want to debug data source access")
     def test_open_dataset(self):
         # Test normal functionality
-        dataset = open_dataset('AEROSOL_AATSR_SU_L3_V4.21_MONTHLY',
-                               '2008-01-01, 2008-03-01')
+        dataset = open_dataset(ds_id='AEROSOL_AATSR_SU_L3_V4.21_MONTHLY',
+                               time_range='2008-01-01, 2008-03-01')
         self.assertIsNotNone(dataset)
 
         # Test swapped dates
         with self.assertRaises(ValueError):
-            open_dataset('AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', '2008-03-01, 2008-01-01')
+            open_dataset(ds_id='AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', time_range='2008-03-01, 2008-01-01')
 
         # Test required arguments
         with self.assertRaises(TypeError):
-            open_dataset('AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', '2008-03-01')
+            open_dataset(ds_id='AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', time_range='2008-03-01')
 
     @unittest.skip(reason="skipped unless you want to debug data source access")
     def test_save_dataset(self):
         # Test normal functionality
-        dataset = open_dataset('AEROSOL_AATSR_SU_L3_V4.21_MONTHLY',
-                               '2008-01-01, 2008-03-01')
+        dataset = open_dataset(ds_id='AEROSOL_AATSR_SU_L3_V4.21_MONTHLY',
+                               time_range='2008-01-01, 2008-03-01')
         save_dataset(dataset, 'remove_me.nc')
         self.assertTrue(os.path.isfile('remove_me.nc'))
         os.remove('remove_me.nc')

test/webapi/test_websocket.py

@@ -28,14 +28,45 @@ def test_get_operations(self):
         ops = self.service.get_operations()
         self.assertIsInstance(ops, list)
         self.assertGreater(len(ops), 20)
+
         self.assertIn('open_dataset', [op['name'] for op in ops])
         open_dataset_op = [op for op in ops if op['name'] == 'open_dataset'][0]
         keys = sorted(list(open_dataset_op.keys()))
         self.assertEqual(keys, ['has_monitor', 'header', 'inputs', 'name', 'outputs', 'qualified_name'])
         keys = sorted(list(open_dataset_op['header'].keys()))
         self.assertEqual(keys, ['description', 'tags'])
         names = [props['name'] for props in open_dataset_op['inputs']]
-        self.assertEqual(names, ['ds_name', 'time_range', 'region', 'var_names', 'normalize',
+        self.assertEqual(names, ['ds_id', 'time_range', 'region', 'var_names', 'normalize',
                                  'force_local', 'local_ds_id'])
         names = [props['name'] for props in open_dataset_op['outputs']]
         self.assertEqual(names, ['return'])
+
+    def test_get_operations_with_deprecations(self):
+        from cate.core.op import op, op_input, op_output, OpRegistry
+
+        registry = OpRegistry()
+
+        @op(registry=registry, deprecated=True)
+        def my_deprecated_op():
+            pass
+
+        @op_input('a', registry=registry)
+        @op_input('b', registry=registry, deprecated=True)
+        @op_output('u', registry=registry, deprecated=True)
+        @op_output('v', registry=registry)
+        def my_op_with_deprecated_io(a, b=None):
+            pass
+
+        self.assertIsNotNone(registry.get_op(my_deprecated_op, fail_if_not_exists=True))
+        self.assertIsNotNone(registry.get_op(my_op_with_deprecated_io, fail_if_not_exists=True))
+
+        ops = self.service.get_operations(registry=registry)
+        op_names = {op['name'] for op in ops}
+        self.assertIn('test.webapi.test_websocket.my_op_with_deprecated_io', op_names)
+        self.assertNotIn('test.webapi.test_websocket.my_deprecated_op', op_names)
+
+        op = [op for op in ops if op['name'] == 'test.webapi.test_websocket.my_op_with_deprecated_io'][0]
+        self.assertEqual(len(op['inputs']), 1)
+        self.assertEqual(op['inputs'][0]['name'], 'a')
+        self.assertEqual(len(op['outputs']), 1)
+        self.assertEqual(op['outputs'][0]['name'], 'v')