moved exception throw in model to test params, udf now drops input ro…

…ws with empty result set, better error message

exasol_transformers_extension/udfs/models/token_classification_udf.py

@@ -120,7 +120,8 @@ def append_predictions_to_input_dataframe(
 
         # Concat predictions and model_df
         pred_df = pd.concat(pred_df_list, axis=0).reset_index(drop=True)
-        model_df = pd.concat([model_df, pred_df], axis=1)
+        model_df = pd.concat([model_df, pred_df], axis=1, join='inner') # join='inner' -> drop rows where results are empty
+
         if self.work_with_spans:
             model_df = self.create_new_span_columns(model_df)
             model_df[["entity_docid", "entity_char_begin", "entity_char_end"]] =\
@@ -141,12 +142,24 @@ def create_dataframes_from_predictions(
         """
         results_df_list = []
         for result in predictions:
-            result_df = pd.DataFrame(result)
-            result_df = result_df[self._desired_fields_in_prediction].rename(
-                columns={
-                    "start": "start_pos",
-                    "end": "end_pos",
-                    "entity_group": "entity"})
+            if result:
+                result_df = pd.DataFrame(result)
+                # need to save before trying to rename, otherwise they get lost and cant be printed in error message
+                result_df_column_names = result_df.columns
+                try:
+                    result_df = result_df[self._desired_fields_in_prediction].rename(
+                        columns={
+                            "start": "start_pos",
+                            "end": "end_pos",
+                            "entity_group": "entity"})
+                except KeyError as e:
+                    # adding more detailed error message
+                    raise KeyError(f"Some expected column was not found in prediction results. "
+                                   f"Expected columns are: {self._desired_fields_in_prediction}. "
+                                   f"Prediction results contain columns: {result_df_column_names}") from e
+            else:
+                # if the result for an input is empty, just append an empty result df, and the input will be dropped during concatenation
+                result_df = pd.DataFrame(result)
             results_df_list.append(result_df)
 
         return results_df_list

tests/unit_tests/udf_wrapper_params/token_classification/error_on_prediction_multiple_model_multiple_batch.py

@@ -38,12 +38,9 @@ class ErrorOnPredictionMultipleModelMultipleBatch:
     work_with_span_output_data = work_with_span_output_data1 + work_with_span_output_data2
 
     tokenizer_model_output_df_model1 =  [make_model_output_for_one_input_row(number_entities=n_entities) * data_size]
-    tokenizer_model_output_df_model2_batch1 =  [make_model_output_for_one_input_row(number_entities=1, #error on pred -> only one output per input
-                                  score=None, start=None, end=None, word=None, entity_group=None,
-                                  )]
-    tokenizer_model_output_df_model2_batch2 =  [make_model_output_for_one_input_row(number_entities=1, #error on pred -> only one output per input
-                                  score=None, start=None, end=None, word=None, entity_group=None,
-                                  )]
+    tokenizer_model_output_df_model2_batch1 =  [[Exception("Traceback mock_pipeline is throwing an error intentionally")]] #error on pred -> only one output per input
+
+    tokenizer_model_output_df_model2_batch2 =  [[Exception("Traceback mock_pipeline is throwing an error intentionally")]]#error on pred -> only one output per input
 
     tokenizer_models_output_df = [tokenizer_model_output_df_model1, tokenizer_model_output_df_model2_batch1, tokenizer_model_output_df_model2_batch2]
 

tests/unit_tests/udf_wrapper_params/token_classification/error_on_prediction_single_model_multiple_batch.py

@@ -28,12 +28,10 @@ class ErrorOnPredictionSingleModelMultipleBatch:
 
     number_complete_batches = data_size // batch_size
     number_remaining_data_entries_in_last_batch = data_size % batch_size
-    tokenizer_model_output_df_model1 = [make_model_output_for_one_input_row(number_entities=1, #error on pred -> only one output per input
-                                  score=None, start=None, end=None, word=None, entity_group=None) * batch_size] * \
+    tokenizer_model_output_df_model1 = [[Exception("Traceback mock_pipeline is throwing an error intentionally")] #error on pred -> only one output per input
+                                   * batch_size] * \
                                 number_complete_batches + \
-                                [make_model_output_for_one_input_row(number_entities=1,
-                                  score=None, start=None, end=None, word=None, entity_group=None,
-                                  ) * number_remaining_data_entries_in_last_batch]
+                                [[Exception("Traceback mock_pipeline is throwing an error intentionally")] * number_remaining_data_entries_in_last_batch]
     tokenizer_models_output_df = [tokenizer_model_output_df_model1]
 
     tmpdir_name = "_".join(("/tmpdir", __qualname__))

tests/unit_tests/udf_wrapper_params/token_classification/error_prediction_containing_only_unknown_fields.py

@@ -19,13 +19,13 @@ class ErrorPredictionOnlyContainsUnknownFields:
     input_data = make_input_row(text_data=text_data) * data_size
     output_data = make_output_row(text_data=text_data,
                                   score=None, start=None, end=None, word=None, entity=None,
-                                  error_msg="Traceback") * n_entities * data_size
+                                  error_msg="Traceback") * data_size # only one output per input
 
     work_with_span_input_data = make_input_row_with_span(text_data=text_data) * data_size
     work_with_span_output_data =  [(bucketfs_conn, sub_dir, model_name,
                                 text_docid, text_start, text_end, agg_strategy_simple,
                                 None, None, None, None, None, None,
-                                "Traceback")] * n_entities * data_size
+                                "Traceback")] * data_size # only one output per input
 
 
     number_complete_batches = data_size // batch_size

tests/unit_tests/udf_wrapper_params/token_classification/error_prediction_missing_expected_field.py

@@ -16,21 +16,29 @@ class ErrorPredictionMissingExpectedFields:
 
     text_data = "error_result_missing_field_'word'" #todo do we want tests for different combinations? seems like a lot of work
     # todo do we want tests for multiple models? multiple inputs where one works and one does not? how many test cases are to many test cases?
-    # todo these should be moved to the base model tests together with the others
+
+    #todo currently we fail and dont return the partial results, only the error message. is this what we want?
 
     input_data = make_input_row(text_data=text_data) * data_size
-    output_data = make_output_row(text_data=text_data, score=None, error_msg="Traceback") * n_entities * data_size
+    output_data = make_output_row(text_data=text_data, score=None,
+                                  start=None, end=None, word=None, entity=None,
+                                  error_msg="Traceback") * data_size # only one output per input
 
     work_with_span_input_data = make_input_row_with_span(text_data=text_data) * data_size
-    work_with_span_output_data =  make_output_row_with_span(score=None,
-                                                            error_msg="Traceback") * n_entities * data_size
+    work_with_span_output_data = [(bucketfs_conn, sub_dir, model_name,
+                                text_docid, text_start, text_end, agg_strategy_simple,
+                                None, None, None, text_docid, None, None,
+                                "Traceback")] * data_size # only one output per input
 
 
     number_complete_batches = data_size // batch_size
     number_remaining_data_entries_in_last_batch = data_size % batch_size
 
-    model_output_row_missing_key = [[model_output_row[0].pop("score")]
-                                    for model_output_row in make_model_output_for_one_input_row(number_entities=n_entities)]
+    model_output_rows = make_model_output_for_one_input_row(number_entities=n_entities)
+    model_output_row_missing_key = []
+    for model_output_row in model_output_rows:
+        removed = model_output_row[0].pop("score")
+        model_output_row_missing_key.append(model_output_row)
 
     tokenizer_model_output_df_model1 = [model_output_row_missing_key * batch_size] * \
                                 number_complete_batches + \

tests/unit_tests/udf_wrapper_params/token_classification/result_contains_additional_keys.py → tests/unit_tests/udf_wrapper_params/token_classification/prediction_contains_additional_keys.py

@@ -4,9 +4,9 @@
     make_output_row, make_input_row_with_span, make_output_row_with_span, bucketfs_conn, \
     text_docid, text_start, text_end, agg_strategy_simple, make_model_output_for_one_input_row, sub_dir, model_name
 
-# todo do we wan to throw in this case? or just ignore additional results?
+# todo do we wan to throw in this case? or just ignore additional results? currently we just ignore
 
-class ErrorPredictionContainsAdditionalFields:
+class PredictionContainsAdditionalFields:
     """
 
     """
@@ -19,15 +19,16 @@ class ErrorPredictionContainsAdditionalFields:
 
     #todod these are not filled out
     input_data = make_input_row(text_data=text_data) * data_size
-    output_data = make_output_row(text_data=text_data, error_msg="Traceback") * n_entities * data_size
+    output_data = make_output_row(text_data=text_data) * n_entities * data_size
 
     work_with_span_input_data = make_input_row_with_span(text_data=text_data) * data_size
-    work_with_span_output_data = make_output_row_with_span(error_msg="Traceback") * n_entities * data_size
+    work_with_span_output_data = make_output_row_with_span() * n_entities * data_size
 
     model_output_rows = make_model_output_for_one_input_row(number_entities=n_entities)
-    model_output_row_wrong_keys = [model_output_row[0].update({"unknown key": "some value", "diff unknown key": 1})
-                                   for model_output_row in model_output_rows]
-    tokenizer_model_output_df_model1 = [model_output_row_wrong_keys * data_size]
+    for model_output_row in model_output_rows:
+        model_output_row[0].update({"unknown key": "some value", "diff unknown key": 1})
+
+    tokenizer_model_output_df_model1 = [model_output_rows * data_size]
     tokenizer_models_output_df = [tokenizer_model_output_df_model1]
 
     tmpdir_name = "_".join(("/tmpdir", __qualname__))

tests/unit_tests/udf_wrapper_params/token_classification/prediction_returns_empty_result.py

@@ -10,24 +10,32 @@ class PredictionReturnsEmptyResult:
 
     """
     expected_model_counter = 1
-    batch_size = 4
-    data_size = 5
+    batch_size = 6
+    data_size = 1
     n_entities = 3
 
     text_data = "error_result_empty"
-    # todo throws error but meassage could be better
     # TODO mention in docu if result is empty row not in output
     input_data = make_input_row() * data_size  + \
-                 make_input_row(text_data=text_data) * data_size
-    output_data = make_output_row() * n_entities * data_size # Result of input #2 is empty, so the row does not appear in the output
+                 make_input_row(text_data=text_data) * data_size  + \
+                 make_input_row() * data_size
+    output_data = make_output_row() * n_entities * data_size + \
+                  make_output_row() * n_entities * data_size# Result of input #2 is empty, so the row does not appear in the output
 
     work_with_span_input_data = make_input_row_with_span() * data_size  + \
-                                make_input_row_with_span(text_data=text_data) * data_size
-    work_with_span_output_data =  make_output_row_with_span() * n_entities * data_size # Result of input #2 is empty, so the row does not appear in the output
+                                make_input_row_with_span(text_data=text_data) * data_size  + \
+                                make_input_row_with_span() * data_size
+
+    work_with_span_output_data =  make_output_row_with_span() * n_entities * data_size  + \
+                                  make_output_row_with_span() * n_entities * data_size # Result of input #2 is empty, so the row does not appear in the output
+
 
     # error on pred -> only one output per input
     tokenizer_model_output_df_model1 = make_model_output_for_one_input_row(number_entities=n_entities) * data_size
-    tokenizer_models_output_df = [tokenizer_model_output_df_model1]
+    tokenizer_model_output_df_model1.append([])
+    tokenizer_model_output_df_model1 = tokenizer_model_output_df_model1 + make_model_output_for_one_input_row(number_entities=n_entities) * data_size
+
+    tokenizer_models_output_df = [[tokenizer_model_output_df_model1]]
 
     tmpdir_name = "_".join(("/tmpdir", __qualname__))
     base_cache_dir1 = PurePosixPath(tmpdir_name, bucketfs_conn)

tests/unit_tests/udfs/test_token_classification.py

@@ -16,8 +16,8 @@
     ErrorPredictionMissingExpectedFields
 from tests.unit_tests.udf_wrapper_params.token_classification.prediction_returns_empty_result import \
     PredictionReturnsEmptyResult
-from tests.unit_tests.udf_wrapper_params.token_classification.result_contains_additional_keys import \
-    ErrorPredictionContainsAdditionalFields
+from tests.unit_tests.udf_wrapper_params.token_classification.prediction_contains_additional_keys import \
+    PredictionContainsAdditionalFields
 from tests.unit_tests.udfs.output_matcher import Output, OutputMatcher
 from tests.utils.mock_bucketfs_location import fake_bucketfs_location_from_conn_object, fake_local_bucketfs_path
 from tests.utils.mock_cast import mock_cast
@@ -63,6 +63,7 @@
     SingleModelSingleBatchIncomplete
 
 
+
 def create_mock_metadata_with_span():
     meta = MockMetaData(
         script_code_wrapper_function=None,
@@ -161,10 +162,9 @@ def create_mock_pipeline_factory(tokenizer_models_output_df, number_of_intended_
     This mock_pipeline is feed into a mock_pipeline_factory.
     """
     mock_pipeline: List[Union[AutoModel, MagicMock]] = [
-        create_autospec(Pipeline, side_effect=tokenizer_models_output_df[i]) if tokenizer_models_output_df[i][0][0][0]["word"]
-        else [Exception("Traceback mock_pipeline is throwing an error intentionally")]
-    for i in range(0, number_of_intended_used_models)
-    ]
+        create_autospec(Pipeline, side_effect=tokenizer_models_output_df[i])
+        for i in range(0, number_of_intended_used_models)
+        ]
 
     mock_pipeline_factory: Union[Pipeline, MagicMock] = create_autospec(Pipeline,
                                                                         side_effect=mock_pipeline)
@@ -210,7 +210,7 @@ def assert_result_matches_expected_output(result, expected_output_data, input_co
     PredictionReturnsEmptyResult,
     ErrorPredictionMissingExpectedFields,
     ErrorPredictionOnlyContainsUnknownFields,
-    ErrorPredictionContainsAdditionalFields
+    PredictionContainsAdditionalFields
 ])
 
 @patch('exasol.python_extension_common.connections.bucketfs_location.create_bucketfs_location_from_conn_object')
@@ -274,7 +274,7 @@ def test_token_classification_with_span(mock_local_path, mock_create_loc, params
     PredictionReturnsEmptyResult,
     ErrorPredictionMissingExpectedFields,
     ErrorPredictionOnlyContainsUnknownFields,
-    ErrorPredictionContainsAdditionalFields
+    PredictionContainsAdditionalFields
 ])
 @patch('exasol.python_extension_common.connections.bucketfs_location.create_bucketfs_location_from_conn_object')
 @patch('exasol_transformers_extension.utils.bucketfs_operations.get_local_bucketfs_path')