From bc765e6ce18c641aa1baa4b95a8fd116f9ca92d6 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Wed, 20 Sep 2017 20:21:06 +0200 Subject: [PATCH] resolves #381 --- cate/cli/main.py | 16 ++++++++++++---- cate/core/op.py | 22 +++++++++++++++++++--- cate/ops/io.py | 16 +++++++++++----- cate/webapi/websocket.py | 13 +++++++++---- scripts/support/openall.py | 2 +- test/cli/test_cli.py | 1 + test/core/test_wsmanag.py | 2 +- test/ops/test_io.py | 12 ++++++------ test/webapi/test_websocket.py | 33 ++++++++++++++++++++++++++++++++- 9 files changed, 92 insertions(+), 25 deletions(-) diff --git a/cate/cli/main.py b/cate/cli/main.py index f78dbeaac..75b76721a 100644 --- a/cate/cli/main.py +++ b/cate/cli/main.py @@ -343,8 +343,12 @@ def _get_op_io_info_str(inputs_or_outputs: dict, title_singular: str, title_plur op_info_str = '' op_info_str += '\n' if inputs_or_outputs: + inputs_or_outputs = {name: properties for name, properties in inputs_or_outputs.items() + if not properties.get('deprecated')} op_info_str += '%s:' % (title_singular if len(inputs_or_outputs) == 1 else title_plural) for name, properties in inputs_or_outputs.items(): + if properties.get('deprecated'): + continue op_info_str += '\n' op_info_str += ' %s (%s)' % (name, _get_op_data_type_str(properties.get('data_type', object))) description = properties.get('description', None) @@ -1029,6 +1033,8 @@ def configure_parser_and_subparsers(cls, parser, subparsers): help="List only operations tagged by TAG or " "that have TAG in one of their tags. " "The comparison is case insensitive.") + list_parser.add_argument('--deprecated', '-d', action='store_true', + help="List deprecated operations.") list_parser.add_argument('--internal', '-i', action='store_true', help='List operations tagged "internal".') list_parser.set_defaults(sub_command_function=cls._execute_list) @@ -1042,11 +1048,13 @@ def configure_parser_and_subparsers(cls, parser, subparsers): def _execute_list(cls, command_args): op_regs = OP_REGISTRY.op_registrations - def _is_op_selected(op_reg, tag_part: str, is_internal: bool): + def _is_op_selected(op_reg, tag_part: str, internal_only: bool, deprecated_only: bool): + if deprecated_only and not op_reg.op_meta_info.header.get('deprecated'): + return False tags = to_list(op_reg.op_meta_info.header.get('tags')) if tags: # Tagged operations - if is_internal: + if internal_only: if 'internal' not in tags: return False else: @@ -1058,13 +1066,13 @@ def _is_op_selected(op_reg, tag_part: str, is_internal: bool): return any(tag_part in tag.lower() for tag in tags) elif isinstance(tags, str): return tag_part in tags.lower() - elif is_internal or tag_part: + elif internal_only or tag_part: # Untagged operations return False return True op_names = sorted([op_name for op_name, op_reg in op_regs.items() if - _is_op_selected(op_reg, command_args.tag, command_args.internal)]) + _is_op_selected(op_reg, command_args.tag, command_args.internal, command_args.deprecated)]) name_pattern = None if command_args.name: name_pattern = command_args.name diff --git a/cate/core/op.py b/cate/core/op.py index 0add92ba2..128939915 100644 --- a/cate/core/op.py +++ b/cate/core/op.py @@ -402,7 +402,11 @@ def __repr__(self): OP_REGISTRY = _DefaultOpRegistry() -def op(registry=OP_REGISTRY, **properties): +def op(tags=UNDEFINED, + version=UNDEFINED, + deprecated=UNDEFINED, + registry=OP_REGISTRY, + **properties): """ ``op`` is a decorator function that registers a Python function or class in the default operation registry or the one given by *registry*, if any. @@ -418,13 +422,21 @@ def op(registry=OP_REGISTRY, **properties): @op(version='X.x') - :param properties: Other properties (keyword arguments) that will be added to the meta-information of operation. + :param tags: An optional list of string tags. + :param version: An optional version string. + :param deprecated: An optional boolean. If set to ``True``, the operation's doc-string should explain why it + has been deprecated and which operation to use instead. :param registry: The operation registry. + :param properties: Other properties (keyword arguments) that will be added to the meta-information of operation. """ def decorator(op_func): + new_properties = dict(tags=tags, + version=version, + deprecated=deprecated, + **properties) op_registration = registry.add_op(op_func, fail_if_exists=False) - op_registration.op_meta_info.header.update({k: v for k, v in properties.items() if v is not UNDEFINED}) + op_registration.op_meta_info.header.update({k: v for k, v in new_properties.items() if v is not UNDEFINED}) return op_registration return decorator @@ -438,6 +450,7 @@ def op_input(input_name: str, value_set_source=UNDEFINED, value_set=UNDEFINED, value_range=UNDEFINED, + deprecated=UNDEFINED, position=UNDEFINED, context=UNDEFINED, registry=OP_REGISTRY, @@ -476,6 +489,8 @@ def op_input(input_name: str, :param value_set: A sequence of the valid values. Note that all values in this sequence must be compatible with *data_type*. :param value_range: A sequence specifying the possible range of valid values. + :param deprecated: An optional boolean. If set to ``True``, the input's doc-string should explain why the input + has been deprecated and which new input to use instead. :param position: The zero-based position of an input. :param context: If ``True``, the value of the operation input will be a dictionary representing the current execution context. For example, @@ -502,6 +517,7 @@ def decorator(op_func): value_set_source=value_set_source, value_set=value_set, value_range=value_range, + deprecated=deprecated, position=position, context=context, **properties) diff --git a/cate/ops/io.py b/cate/ops/io.py index 32c588ca0..725b7484e 100644 --- a/cate/ops/io.py +++ b/cate/ops/io.py @@ -38,7 +38,8 @@ @op(tags=['input']) -@op_input('ds_name') +@op_input('ds_id') +@op_input('ds_name', deprecated=True) @op_input('time_range', data_type=TimeRangeLike) @op_input('region', data_type=PolygonLike) @op_input('var_names', data_type=VarNamesLike) @@ -46,6 +47,7 @@ @op_input('force_local') @op_input('local_ds_id') def open_dataset(ds_name: str, + ds_id: str = None, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, @@ -56,7 +58,8 @@ def open_dataset(ds_name: str, """ Open a dataset from a data source identified by *ds_name*. - :param ds_name: The name of data source. + :param ds_name: The name of data source. This parameter has been deprecated, please use *ds_id* instead. + :param ds_id: The identifier for the data source. :param time_range: Optional time range of the requested dataset :param region: Optional spatial region of the requested dataset :param var_names: Optional names of variables of the requested dataset @@ -67,9 +70,12 @@ def open_dataset(ds_name: str, :return: An new dataset instance. """ import cate.core.ds - ds = cate.core.ds.open_dataset(data_source=ds_name, time_range=time_range, - var_names=var_names, region=region, - force_local=force_local, local_ds_id=local_ds_id, + ds = cate.core.ds.open_dataset(data_source=ds_id or ds_name, + time_range=time_range, + var_names=var_names, + region=region, + force_local=force_local, + local_ds_id=local_ds_id, monitor=monitor) if ds and normalize: return normalize_op(ds) diff --git a/cate/webapi/websocket.py b/cate/webapi/websocket.py index e0b2dc396..84ccfcf18 100644 --- a/cate/webapi/websocket.py +++ b/cate/webapi/websocket.py @@ -232,18 +232,23 @@ def remove_local_datasource(self, data_source_name: str, remove_files: bool) -> data_store.remove_data_source(data_source_name, remove_files) return self.get_data_sources('local', monitor=Monitor.NONE) - def get_operations(self) -> List[dict]: + def get_operations(self, registry=None) -> List[dict]: """ Get registered operations. :return: JSON-serializable list of data sources, sorted by name. """ + registry = registry or OP_REGISTRY op_list = [] - for op_name, op_reg in OP_REGISTRY.op_registrations.items(): + for op_name, op_reg in registry.op_registrations.items(): + if op_reg.op_meta_info.header.get('deprecated'): + continue op_json_dict = op_reg.op_meta_info.to_json_dict() op_json_dict['name'] = op_name - op_json_dict['inputs'] = [dict(name=name, **props) for name, props in op_json_dict['inputs'].items()] - op_json_dict['outputs'] = [dict(name=name, **props) for name, props in op_json_dict['outputs'].items()] + op_json_dict['inputs'] = [dict(name=name, **props) for name, props in op_json_dict['inputs'].items() + if not props.get('deprecated')] + op_json_dict['outputs'] = [dict(name=name, **props) for name, props in op_json_dict['outputs'].items() + if not props.get('deprecated')] op_list.append(op_json_dict) return sorted(op_list, key=lambda op: op['name']) diff --git a/scripts/support/openall.py b/scripts/support/openall.py index 560b3f7ba..93d7a2a9c 100644 --- a/scripts/support/openall.py +++ b/scripts/support/openall.py @@ -23,7 +23,7 @@ def test_data_source(data_source, out_dir, result): dataset = None t0 = time.clock() try: - dataset = open_dataset(data_source_id, + dataset = open_dataset(ds_id=data_source_id, force_local=True, monitor=ConsoleMonitor(stay_in_line=True, progress_bar_size=60)) diff --git a/test/cli/test_cli.py b/test/cli/test_cli.py index 7d0cecbe4..5a2b7517f 100644 --- a/test/cli/test_cli.py +++ b/test/cli/test_cli.py @@ -285,6 +285,7 @@ def test_op_list(self): self.assert_main(['op', 'list', '--internal'], expected_stdout=['4 operations found']) self.assert_main(['op', 'list', '--tag', 'input'], expected_stdout=['8 operations found']) self.assert_main(['op', 'list', '--tag', 'output'], expected_stdout=['6 operations found']) + self.assert_main(['op', 'list', '--deprecated'], expected_stdout=['No operations found']) @unittest.skip(reason='Hardcoded values from remote service, contains outdated assumptions') diff --git a/test/core/test_wsmanag.py b/test/core/test_wsmanag.py index 7ae9c3bf3..8d3646769 100644 --- a/test/core/test_wsmanag.py +++ b/test/core/test_wsmanag.py @@ -185,7 +185,7 @@ def test_session(self): self.del_base_dir(base_dir) - def test_persitence(self): + def test_persistence(self): base_dir = self.new_base_dir('TESTOMAT') workspace_manager = self.new_workspace_manager() diff --git a/test/ops/test_io.py b/test/ops/test_io.py index d888afd2d..50c6bbd23 100644 --- a/test/ops/test_io.py +++ b/test/ops/test_io.py @@ -14,23 +14,23 @@ class TestIO(TestCase): @unittest.skip(reason="skipped unless you want to debug data source access") def test_open_dataset(self): # Test normal functionality - dataset = open_dataset('AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', - '2008-01-01, 2008-03-01') + dataset = open_dataset(ds_id='AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', + time_range='2008-01-01, 2008-03-01') self.assertIsNotNone(dataset) # Test swapped dates with self.assertRaises(ValueError): - open_dataset('AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', '2008-03-01, 2008-01-01') + open_dataset(ds_id='AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', time_range='2008-03-01, 2008-01-01') # Test required arguments with self.assertRaises(TypeError): - open_dataset('AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', '2008-03-01') + open_dataset(ds_id='AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', time_range='2008-03-01') @unittest.skip(reason="skipped unless you want to debug data source access") def test_save_dataset(self): # Test normal functionality - dataset = open_dataset('AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', - '2008-01-01, 2008-03-01') + dataset = open_dataset(ds_id='AEROSOL_AATSR_SU_L3_V4.21_MONTHLY', + time_range='2008-01-01, 2008-03-01') save_dataset(dataset, 'remove_me.nc') self.assertTrue(os.path.isfile('remove_me.nc')) os.remove('remove_me.nc') diff --git a/test/webapi/test_websocket.py b/test/webapi/test_websocket.py index 7d24e4cae..78d696d37 100644 --- a/test/webapi/test_websocket.py +++ b/test/webapi/test_websocket.py @@ -28,6 +28,7 @@ def test_get_operations(self): ops = self.service.get_operations() self.assertIsInstance(ops, list) self.assertGreater(len(ops), 20) + self.assertIn('open_dataset', [op['name'] for op in ops]) open_dataset_op = [op for op in ops if op['name'] == 'open_dataset'][0] keys = sorted(list(open_dataset_op.keys())) @@ -35,7 +36,37 @@ def test_get_operations(self): keys = sorted(list(open_dataset_op['header'].keys())) self.assertEqual(keys, ['description', 'tags']) names = [props['name'] for props in open_dataset_op['inputs']] - self.assertEqual(names, ['ds_name', 'time_range', 'region', 'var_names', 'normalize', + self.assertEqual(names, ['ds_id', 'time_range', 'region', 'var_names', 'normalize', 'force_local', 'local_ds_id']) names = [props['name'] for props in open_dataset_op['outputs']] self.assertEqual(names, ['return']) + + def test_get_operations_with_deprecations(self): + from cate.core.op import op, op_input, op_output, OpRegistry + + registry = OpRegistry() + + @op(registry=registry, deprecated=True) + def my_deprecated_op(): + pass + + @op_input('a', registry=registry) + @op_input('b', registry=registry, deprecated=True) + @op_output('u', registry=registry, deprecated=True) + @op_output('v', registry=registry) + def my_op_with_deprecated_io(a, b=None): + pass + + self.assertIsNotNone(registry.get_op(my_deprecated_op, fail_if_not_exists=True)) + self.assertIsNotNone(registry.get_op(my_op_with_deprecated_io, fail_if_not_exists=True)) + + ops = self.service.get_operations(registry=registry) + op_names = {op['name'] for op in ops} + self.assertIn('test.webapi.test_websocket.my_op_with_deprecated_io', op_names) + self.assertNotIn('test.webapi.test_websocket.my_deprecated_op', op_names) + + op = [op for op in ops if op['name'] == 'test.webapi.test_websocket.my_op_with_deprecated_io'][0] + self.assertEqual(len(op['inputs']), 1) + self.assertEqual(op['inputs'][0]['name'], 'a') + self.assertEqual(len(op['outputs']), 1) + self.assertEqual(op['outputs'][0]['name'], 'v')