performance metrics

jdvelasq · Jan 23, 2025 · 2669ce2 · 2669ce2
1 parent 8173fe4
commit 2669ce2
Show file tree

Hide file tree

Showing 139 changed files with 692 additions and 261 deletions.
diff --git a/sphinx/index.rst b/sphinx/index.rst
@@ -24,7 +24,6 @@ Techminer 2+
     :caption: PREPARE
     :maxdepth: 1
 
-    prepare/database/__index__
     prepare/fields/__index__
     prepare/thesaurus/__index__
     prepare/transform/__index__

diff --git a/sphinx/prepare/database/__index__.rst b/sphinx/prepare/database/__index__.rst
diff --git a/sphinx/prepare/database/coverage.rst b/sphinx/prepare/database/coverage.rst
diff --git a/sphinx/prepare/database/query.rst b/sphinx/prepare/database/query.rst
diff --git a/sphinx/prepare/database/statistics.rst b/sphinx/prepare/database/statistics.rst
diff --git a/sphinx/prepare/database/summary_sheet.rst b/sphinx/prepare/database/summary_sheet.rst
diff --git a/techminer2/analyze/citation_network/others/internals/create_citation_nx_graph.py b/techminer2/analyze/citation_network/others/internals/create_citation_nx_graph.py
@@ -10,7 +10,9 @@
 import numpy as np
 
 from .....database.load.load__filtered_database import load__filtered_database
-from ....metrics.performance_metrics_dataframe import performance_metrics_frame
+from .....database.metrics.performance_metrics.dataframe import (
+    performance_metrics_frame,
+)
 
 
 def _create_citation_nx_graph(

diff --git a/techminer2/analyze/coupling_network/others/internals/create_coupling_nx_graph.py b/techminer2/analyze/coupling_network/others/internals/create_coupling_nx_graph.py
@@ -10,10 +10,12 @@
 import numpy as np
 
 from .....database.load.load__filtered_database import load__filtered_database
+from .....database.metrics.performance_metrics.dataframe import (
+    performance_metrics_frame,
+)
 from .....internals.utils.utils_append_occurrences_and_citations_to_axis import (
     _utils_append_occurrences_and_citations_to_axis,
 )
-from ....metrics.performance_metrics_dataframe import performance_metrics_frame
 
 
 def _create_coupling_nx_graph(

diff --git a/techminer2/analyze/cross_co_occurrence/dataframe.py b/techminer2/analyze/cross_co_occurrence/dataframe.py
@@ -52,6 +52,7 @@
 
 """
 from ...database.load.load__filtered_database import load__filtered_database
+from ...database.load.load__user_stopwords import load__user_stopwords
 from ...internals.mt.mt_calculate_global_performance_metrics import (
     _mt_calculate_global_performance_metrics,
 )
@@ -65,7 +66,6 @@
     DatabaseFilters,
     SetDatabaseFiltersMixin,
 )
-from ...internals.stopwords.load_user_stopwords import load_user_stopwords
 from ...internals.utils.utils_compute_occurrences_and_citations import (
     _utils_compute_occurrences_and_citations,
 )
@@ -138,7 +138,7 @@ def filter_terms(
         raw_matrix_list = raw_matrix_list.rename(columns={columns: "column"})
         raw_matrix_list = raw_matrix_list.assign(row=records[[rows]])
 
-        stopwords = load_user_stopwords(root_dir=root_dir)
+        stopwords = load__user_stopwords(root_dir=root_dir)
 
         for name in ["column", "row"]:
             raw_matrix_list[name] = raw_matrix_list[name].str.split(";")

diff --git a/techminer2/analyze/metrics/__init__.py b/techminer2/analyze/metrics/__init__.py
@@ -1,10 +1,10 @@
 """Metrics menu options"""
 
+from ...database.metrics.performance_metrics.dataframe import performance_metrics_frame
 from ..collaboration.collaboration_metrics_dataframe import collaboration_metrics_frame
 from ..collaboration.collaboration_metrics_plot import collaboration_metrics_plot
 from .general_metrics_dataframe import general_metrics_frame
 from .growth_metrics_dataframe import growth_metrics_frame
-from .performance_metrics_dataframe import performance_metrics_frame
 from .terms_by_year.dataframe import terms_by_year_frame
 from .terms_by_year.ranking_plot import terms_by_year_plot
 from .tfidf_dataframe import tfidf_frame

diff --git a/techminer2/analyze/metrics/performance_metrics_dataframe.py b/techminer2/analyze/metrics/performance_metrics_dataframe.py
diff --git a/techminer2/database/internals/__init__.py → techminer2/database/deprecated/__init__.py b/techminer2/database/internals/__init__.py → techminer2/database/deprecated/__init__.py
diff --git a/...recated/create_imported_records_report.py → ...recated/create_imported_records_report.py b/...recated/create_imported_records_report.py → ...recated/create_imported_records_report.py
diff --git a/...ls/deprecated/extract_meaningful_words.py → ...se/deprecated/extract_meaningful_words.py b/...ls/deprecated/extract_meaningful_words.py → ...se/deprecated/extract_meaningful_words.py
diff --git a/...nternals/deprecated/list_cleanup_words.py → ...database/deprecated/list_cleanup_words.py b/...nternals/deprecated/list_cleanup_words.py → ...database/deprecated/list_cleanup_words.py
diff --git a/...ted/records_with_no_abstract_available.py → ...ted/records_with_no_abstract_available.py b/...ted/records_with_no_abstract_available.py → ...ted/records_with_no_abstract_available.py
diff --git a/...nternals/deprecated/review_nlp_phrases.py → ...database/deprecated/review_nlp_phrases.py b/...nternals/deprecated/review_nlp_phrases.py → ...database/deprecated/review_nlp_phrases.py
diff --git a/...e/internals/deprecated/update_abstract.py → ...r2/database/deprecated/update_abstract.py b/...e/internals/deprecated/update_abstract.py → ...r2/database/deprecated/update_abstract.py
diff --git a/techminer2/database/field_extractors/extractors__contains.py b/techminer2/database/field_extractors/extractors__contains.py
@@ -35,7 +35,7 @@
 
 """
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_extractors.internal__contains import internal__contains
+from .internals.internal__contains import internal__contains
 
 
 class ContainsExtractor(

diff --git a/techminer2/database/field_extractors/extractors__ends_with.py b/techminer2/database/field_extractors/extractors__ends_with.py
@@ -38,7 +38,7 @@
  
 """
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_extractors.internal__ends_with import internal__ends_with
+from .internals.internal__ends_with import internal__ends_with
 
 
 class EndsWithExtractor(

diff --git a/techminer2/database/field_extractors/extractors__fields_difference.py b/techminer2/database/field_extractors/extractors__fields_difference.py
@@ -39,9 +39,7 @@
 """
 
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_extractors.internal__fields_difference import (
-    internal__fields_difference,
-)
+from .internals.internal__fields_difference import internal__fields_difference
 
 
 class FieldsDifferenceExtractor(

diff --git a/techminer2/database/field_extractors/extractors__fields_intersection.py b/techminer2/database/field_extractors/extractors__fields_intersection.py
@@ -39,9 +39,7 @@
 """
 
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_extractors.internal__fields_intersection import (
-    internal__fields_intersection,
-)
+from .internals.internal__fields_intersection import internal__fields_intersection
 
 
 class FieldsIntersectionExtractor(

diff --git a/techminer2/database/field_extractors/extractors__full_match.py b/techminer2/database/field_extractors/extractors__full_match.py
@@ -30,7 +30,7 @@
 
 """
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_extractors.internal__full_match import internal__full_match
+from .internals.internal__full_match import internal__full_match
 
 
 class FullMatchExtractor(

diff --git a/techminer2/database/field_extractors/extractors__match.py b/techminer2/database/field_extractors/extractors__match.py
@@ -32,7 +32,7 @@
 """
 
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_extractors.internal__match import internal__match
+from .internals.internal__match import internal__match
 
 
 class MatchExtractor(

diff --git a/techminer2/database/field_extractors/extractors__starts_with.py b/techminer2/database/field_extractors/extractors__starts_with.py
@@ -41,7 +41,7 @@
 """
 
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_extractors.internal__starts_with import internal__starts_with
+from .internals.internal__starts_with import internal__starts_with
 
 
 class StartsWithExtractor(

diff --git a/techminer2/database/field_extractors/extractors__stemming_and.py b/techminer2/database/field_extractors/extractors__stemming_and.py
@@ -36,7 +36,7 @@
 """
 
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_extractors.internal__stemming import internal__stemming_and
+from .internals.internal__stemming import internal__stemming_and
 
 
 class StemmingAndExtractor(

diff --git a/techminer2/database/field_extractors/extractors__stemming_or.py b/techminer2/database/field_extractors/extractors__stemming_or.py
@@ -44,7 +44,7 @@
 """
 
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_extractors.internal__stemming import internal__stemming_or
+from .internals.internal__stemming import internal__stemming_or
 
 
 class StemmingOrExtractor(

diff --git a/techminer2/database/field_extractors/extractors__top_terms.py b/techminer2/database/field_extractors/extractors__top_terms.py
@@ -43,7 +43,26 @@
 
 """
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_extractors.internal__top_terms import internal__top_terms
+from ..load import load__filtered_database
+from ..metrics.performance_metrics.internals.internal__add_rank_field_by_metrics import (
+    internal__add_rank_field_by_metrics,
+)
+from ..metrics.performance_metrics.internals.internal__check_field_types import (
+    internal__check_field_types,
+)
+from ..metrics.performance_metrics.internals.internal__compute_basic_metrics_per_term_fields import (
+    internal__compute_basic_metrics_per_term_fields,
+)
+from ..metrics.performance_metrics.internals.internal__explode_terms_in_field import (
+    internal__explode_terms_in_field,
+)
+from ..metrics.performance_metrics.internals.internal__remove_stopwords_from_axis import (
+    internal__remove_stopwords_from_axis,
+)
+from ..metrics.performance_metrics.internals.internal__select_fields import (
+    internal__select_fields,
+)
+from .internals.internal__top_terms import internal__top_terms
 
 
 class TopTermsExtractor(
@@ -53,6 +72,44 @@ class TopTermsExtractor(
 
     def build(self):
 
+        records = load__filtered_database(
+            root_dir=self.params.root_dir,
+            database=self.params.database,
+            record_years_range=self.params.record_years_range,
+            record_citations_range=self.params.record_citations_range,
+            records_order_by=self.params.records_order_by,
+            records_match=self.params.records_match,
+        )
+
+        records = internal__select_fields(
+            records=records,
+            field=self.params.source_field,
+        )
+
+        records = internal__explode_terms_in_field(
+            records=records,
+            field=self.params.source_field,
+        )
+
+        records = internal__compute_basic_metrics_per_term_fields(
+            records=records,
+            field=self.params.source_field,
+        )
+
+        records = internal__remove_stopwords_from_axis(
+            dataframe=records,
+            root_dir=self.params.root_dir,
+            axis=0,
+        )
+
+        records = internal__add_rank_field_by_metrics(
+            records=records,
+        )
+
+        records = internal__check_field_types(
+            records=records,
+        )
+
         return internal__top_terms(
             source_field=self.params.source_field,
             terms_order_criteria=self.params.terms_order_by,

diff --git a/...database/internals/deprecated/__init__.py → ...se/field_extractors/internals/__init__.py b/...database/internals/deprecated/__init__.py → ...se/field_extractors/internals/__init__.py
diff --git a/...ls/field_extractors/internal__contains.py → ...xtractors/internals/internal__contains.py b/...ls/field_extractors/internal__contains.py → ...xtractors/internals/internal__contains.py
diff --git a/...s/field_extractors/internal__ends_with.py → ...tractors/internals/internal__ends_with.py b/...s/field_extractors/internal__ends_with.py → ...tractors/internals/internal__ends_with.py
diff --git a/...d_extractors/internal__extract_country.py → ...rs/internals/internal__extract_country.py b/...d_extractors/internal__extract_country.py → ...rs/internals/internal__extract_country.py
diff --git a/...ors/internal__extract_meaningful_words.py → ...als/internal__extract_meaningful_words.py b/...ors/internal__extract_meaningful_words.py → ...als/internal__extract_meaningful_words.py
diff --git a/...extractors/internal__fields_difference.py → .../internals/internal__fields_difference.py b/...extractors/internal__fields_difference.py → .../internals/internal__fields_difference.py
diff --git a/...tractors/internal__fields_intersection.py → ...nternals/internal__fields_intersection.py b/...tractors/internal__fields_intersection.py → ...nternals/internal__fields_intersection.py
diff --git a/.../field_extractors/internal__full_match.py → ...ractors/internals/internal__full_match.py b/.../field_extractors/internal__full_match.py → ...ractors/internals/internal__full_match.py
diff --git a/...ternal__get_field_values_from_database.py → ...ternal__get_field_values_from_database.py b/...ternal__get_field_values_from_database.py → ...ternal__get_field_values_from_database.py
diff --git a/...rnals/field_extractors/internal__match.py → ...d_extractors/internals/internal__match.py b/...rnals/field_extractors/internal__match.py → ...d_extractors/internals/internal__match.py
diff --git a/...field_extractors/internal__starts_with.py → ...actors/internals/internal__starts_with.py b/...field_extractors/internal__starts_with.py → ...actors/internals/internal__starts_with.py
diff --git a/...ls/field_extractors/internal__stemming.py → ...xtractors/internals/internal__stemming.py b/...ls/field_extractors/internal__stemming.py → ...xtractors/internals/internal__stemming.py
diff --git a/...s/field_extractors/internal__top_terms.py → ...tractors/internals/internal__top_terms.py b/...s/field_extractors/internal__top_terms.py → ...tractors/internals/internal__top_terms.py
diff --git a/techminer2/database/field_operators/operators__clean_text.py b/techminer2/database/field_operators/operators__clean_text.py
@@ -20,7 +20,7 @@
 
 """
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_operators.internal__clean_text import internal__clean_text
+from ..ingest.internals.operators.internal__clean_text import internal__clean_text
 from .operators__protected_fields import PROTECTED_FIELDS
 
 

diff --git a/techminer2/database/field_operators/operators__collect_nouns_and_phrases.py b/techminer2/database/field_operators/operators__collect_nouns_and_phrases.py
@@ -20,7 +20,7 @@
 
 """
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_operators.internal__collect_nouns_and_phrases import (
+from ..ingest.internals.operators.internal__collect_nouns_and_phrases import (
     internal__collect_nouns_and_phrases,
 )
 from .operators__protected_fields import PROTECTED_FIELDS

diff --git a/techminer2/database/field_operators/operators__copy_field.py b/techminer2/database/field_operators/operators__copy_field.py
@@ -20,7 +20,7 @@
 
 """
 from ...internals.mixins import InputFunctionsMixin
-from ..internals.field_operators.internal__copy_field import internal__copy_field
+from ..ingest.internals.operators.internal__copy_field import internal__copy_field
 from .operators__protected_fields import PROTECTED_FIELDS