[SYSTEMDS-3701] Rework Modality Data Structure

src/main/python/systemds/scuro/__init__.py

@@ -18,59 +18,60 @@
 # under the License.
 #
 # -------------------------------------------------------------
+from systemds.scuro.dataloader.base_loader import BaseLoader
+from systemds.scuro.dataloader.audio_loader import AudioLoader
+from systemds.scuro.dataloader.video_loader import VideoLoader
+from systemds.scuro.dataloader.text_loader import TextLoader
+from systemds.scuro.dataloader.json_loader import JSONLoader
 from systemds.scuro.representations.representation import Representation
 from systemds.scuro.representations.average import Average
 from systemds.scuro.representations.concatenation import Concatenation
-from systemds.scuro.representations.fusion import Fusion
 from systemds.scuro.representations.sum import Sum
 from systemds.scuro.representations.max import RowMax
 from systemds.scuro.representations.multiplication import Multiplication
 from systemds.scuro.representations.mel_spectrogram import MelSpectrogram
 from systemds.scuro.representations.resnet import ResNet
 from systemds.scuro.representations.bert import Bert
-from systemds.scuro.representations.unimodal import UnimodalRepresentation
 from systemds.scuro.representations.lstm import LSTM
-from systemds.scuro.representations.representation_dataloader import (
-    NPY,
-    Pickle,
-    HDF5,
-    JSON,
-)
+from systemds.scuro.representations.bow import BoW
+from systemds.scuro.representations.glove import GloVe
+from systemds.scuro.representations.tfidf import TfIdf
+from systemds.scuro.representations.word2vec import W2V
 from systemds.scuro.models.model import Model
 from systemds.scuro.models.discrete_model import DiscreteModel
-from systemds.scuro.modality.aligned_modality import AlignedModality
-from systemds.scuro.modality.audio_modality import AudioModality
-from systemds.scuro.modality.video_modality import VideoModality
-from systemds.scuro.modality.text_modality import TextModality
 from systemds.scuro.modality.modality import Modality
+from systemds.scuro.modality.unimodal_modality import UnimodalModality
+from systemds.scuro.modality.transformed import TransformedModality
+from systemds.scuro.modality.type import ModalityType
 from systemds.scuro.aligner.dr_search import DRSearch
 from systemds.scuro.aligner.task import Task
 
 
 __all__ = [
+    "BaseLoader",
+    "AudioLoader",
+    "VideoLoader",
+    "TextLoader",
     "Representation",
     "Average",
     "Concatenation",
-    "Fusion",
     "Sum",
     "RowMax",
     "Multiplication",
     "MelSpectrogram",
     "ResNet",
     "Bert",
-    "UnimodalRepresentation",
     "LSTM",
-    "NPY",
-    "Pickle",
-    "HDF5",
-    "JSON",
+    "BoW",
+    "GloVe",
+    "TfIdf",
+    "W2V",
     "Model",
     "DiscreteModel",
-    "AlignedModality",
-    "AudioModality",
-    "VideoModality",
-    "TextModality",
     "Modality",
+    "UnimodalModality",
+    "TransformedModality",
+    "ModalityType",
     "DRSearch",
     "Task",
 ]

src/main/python/systemds/scuro/aligner/alignment.py

@@ -19,7 +19,6 @@
 #
 # -------------------------------------------------------------
 from aligner.alignment_strategy import AlignmentStrategy
-from modality.aligned_modality import AlignedModality
 from modality.modality import Modality
 from modality.representation import Representation
 from aligner.similarity_measures import Measure
@@ -46,4 +45,4 @@ def __init__(
         self.similarity_measure = similarity_measure
 
     def align_modalities(self) -> Modality:
-        return AlignedModality(Representation())
+        return Modality(Representation())

src/main/python/systemds/scuro/aligner/dr_search.py

@@ -23,7 +23,6 @@
 from typing import List
 
 from systemds.scuro.aligner.task import Task
-from systemds.scuro.modality.aligned_modality import AlignedModality
 from systemds.scuro.modality.modality import Modality
 from systemds.scuro.representations.representation import Representation
 
@@ -64,27 +63,25 @@ def __init__(
 
     def set_best_params(
         self,
-        modality_name: str,
         representation: Representation,
         scores: List[float],
         modality_names: List[str],
     ):
         """
         Updates the best parameters for given modalities, representation, and score
-        :param modality_name: The name of the aligned modality
         :param representation: The representation used to retrieve the current score
-        :param score: achieved score for the set of modalities and representation
+        :param scores: achieved train/test scores for the set of modalities and representation
         :param modality_names: List of modality names used in this setting
         :return:
         """
 
         # check if modality name is already in dictionary
-        if modality_name not in self.scores.keys():
+        if "_".join(modality_names) not in self.scores.keys():
             # if not add it to dictionary
-            self.scores[modality_name] = {}
+            self.scores["_".join(modality_names)] = {}
 
         # set score for representation
-        self.scores[modality_name][representation] = scores
+        self.scores["_".join(modality_names)][representation] = scores
 
         # compare current score with best score
         if scores[1] > self.best_score:
@@ -113,13 +110,12 @@ def fit_random(self, seed=-1):
         modality_combination = random.choice(modalities)
         representation = random.choice(self.representations)
 
-        modality = AlignedModality(representation, list(modality_combination))  # noqa
-        modality.combine()
+        modality = modality_combination[0].combine(
+            modality_combination[1:], representation
+        )
 
         scores = self.task.run(modality.data)
-        self.set_best_params(
-            modality.name, representation, scores, modality.get_modality_names()
-        )
+        self.set_best_params(representation, scores, modality.get_modality_names())
 
         return self.best_representation, self.best_score, self.best_modalities
 
@@ -133,14 +129,14 @@ def fit_enumerate_all(self):
         for M in range(1, len(self.modalities) + 1):
             for combination in itertools.combinations(self.modalities, M):
                 for representation in self.representations:
-                    modality = AlignedModality(
-                        representation, list(combination)
-                    )  # noqa
-                    modality.combine()
+                    modality = combination[0]
+                    if len(combination) > 1:
+                        modality = combination[0].combine(
+                            list(combination[1:]), representation
+                        )
 
                     scores = self.task.run(modality.data)
                     self.set_best_params(
-                        modality.name,
                         representation,
                         scores,
                         modality.get_modality_names(),
@@ -164,7 +160,8 @@ def transform(self, modalities: List[Modality]):
         for modality_name in self.best_modalities:
             used_modalities.append(get_modalities_by_name(modalities, modality_name))
 
-        modality = AlignedModality(self.best_representation, used_modalities)  # noqa
-        modality.combine(self.task.train_indices)
+        modality = used_modalities[0].combine(
+            used_modalities[1:], self.best_representation
+        )
 
         return modality.data

src/main/python/systemds/scuro/dataloader/__init__.py

@@ -0,0 +1,20 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------

src/main/python/systemds/scuro/dataloader/audio_loader.py

@@ -0,0 +1,39 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+from typing import List, Optional
+
+import librosa
+from systemds.scuro.dataloader.base_loader import BaseLoader
+
+
+class AudioLoader(BaseLoader):
+    def __init__(
+        self,
+        source_path: str,
+        indices: List[str],
+        chunk_size: Optional[int] = None,
+    ):
+        super().__init__(source_path, indices, chunk_size)
+
+    def extract(self, file: str):
+        self.file_sanity_check(file)
+        audio, sr = librosa.load(file)
+        self.data.append(audio)

src/main/python/systemds/scuro/dataloader/base_loader.py

@@ -0,0 +1,92 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+import os
+from abc import ABC, abstractmethod
+from typing import List, Optional, Union
+
+
+class BaseLoader(ABC):
+    def __init__(
+        self, source_path: str, indices: List[str], chunk_size: Optional[int] = None
+    ):
+        """
+        Base class to load raw data for a given list of indices and stores them in the data object
+        :param source_path: The location where the raw data lies
+        :param indices: A list of indices as strings that are corresponding to the file names
+        :param chunk_size: An optional argument to load the data in chunks instead of all at once
+        (otherwise please provide your own Dataloader that knows about the file name convention)
+        """
+        self.data = []
+        self.source_path = source_path
+        self.indices = indices
+        self.chunk_size = chunk_size
+        self.next_chunk = 0
+
+        if self.chunk_size:
+            self.num_chunks = int(len(self.indices) / self.chunk_size)
+
+    def load(self):
+        """
+        Takes care of loading the raw data either chunk wise (if chunk size is defined) or all at once
+        """
+        if self.chunk_size:
+            return self._load_next_chunk()
+
+        return self._load(self.indices)
+
+    def _load_next_chunk(self):
+        """
+        Loads the next chunk of data
+        """
+        self.data = []
+        next_chunk_indices = self.indices[
+            self.next_chunk * self.chunk_size : (self.next_chunk + 1) * self.chunk_size
+        ]
+        self.next_chunk += 1
+        return self._load(next_chunk_indices)
+
+    def _load(self, indices: List[str]):
+        is_dir = True if os.path.isdir(self.source_path) else False
+
+        if is_dir:
+            _, ext = os.path.splitext(os.listdir(self.source_path)[0])
+            for index in indices:
+                self.extract(self.source_path + index + ext)
+        else:
+            self.extract(self.source_path, indices)
+
+        return self.data
+
+    @abstractmethod
+    def extract(self, file: str, index: Optional[Union[str, List[str]]] = None):
+        pass
+
+    def file_sanity_check(self, file):
+        """
+        Checks if the file can be found is not empty
+        """
+        try:
+            file_size = os.path.getsize(file)
+        except:
+            raise (f"Error: File {0} not found!".format(file))
+
+        if file_size == 0:
+            raise ("File {0} is empty".format(file))

src/main/python/systemds/scuro/dataloader/json_loader.py

@@ -0,0 +1,43 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+import json
+
+from systemds.scuro.dataloader.base_loader import BaseLoader
+from typing import Optional, List
+
+
+class JSONLoader(BaseLoader):
+    def __init__(
+        self,
+        source_path: str,
+        indices: List[str],
+        field: str,
+        chunk_size: Optional[int] = None,
+    ):
+        super().__init__(source_path, indices, chunk_size)
+        self.field = field
+
+    def extract(self, file: str, indices: List[str]):
+        self.file_sanity_check(file)
+        with open(file) as f:
+            json_file = json.load(f)
+            for idx in indices:
+                self.data.append(json_file[idx][self.field])