From 1dac2fe49605bfb7688157989c0cc408955f1744 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 2 Sep 2021 11:46:50 +0200 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20REFACTOR:=20Remove=20`Quer?= =?UTF-8?q?yManager`=20(#5101)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `AbstractQueryManager` class and its implementations, `DjangoQueryManager` and `SqlaQueryManager`, is an odd mix of a few methods, with no real conceptual purpose; essentially a catch-all for things that didn't have another home. In this PR, the classes are removed and their methods are reassigned as appropriate: * `get_creation_statistics` has been moved to the `BackendQueryBuilder` * `get_duplicate_uuids` and `apply_new_uuid_mapping` have been moved to `aiida/backends/general/migrations/duplicate_uuids.py`. They were defined both in `aiida/backends/general/migrations/utils.py` as well as in `aiida/manage/database/integrity/duplicate_uuid.py`. Note that the former had been adapted after the repository redesign, but the latter was omitted by mistake. The latter is now correctly updated to contain compatibility for the new repository. Since the methods directly use SQL, they cannot be easily abstracted and moved to the `BackendQueryBuilder`. * `get_bands_and_parents_structure` has been moved to the ORM module in `aiida/orm/nodes/data/array/bands.py`. --- .../0014_add_node_uuid_unique_constraint.py | 2 +- .../djsite/db/migrations/0018_django_1_11.py | 2 +- aiida/backends/djsite/queries.py | 229 --------------- aiida/backends/general/abstractqueries.py | 265 ------------------ .../general/migrations/duplicate_uuids.py} | 41 ++- aiida/backends/general/migrations/utils.py | 96 ------- ...f3d4882837_make_all_uuid_columns_unique.py | 3 +- aiida/backends/sqlalchemy/queries.py | 70 ----- aiida/cmdline/commands/cmd_data/cmd_bands.py | 6 +- aiida/cmdline/commands/cmd_database.py | 4 +- aiida/manage/__init__.py | 4 - aiida/manage/database/__init__.py | 4 - aiida/manage/database/integrity/__init__.py | 5 - aiida/orm/implementation/backends.py | 6 - aiida/orm/implementation/django/backend.py | 6 - aiida/orm/implementation/querybuilder.py | 23 +- .../orm/implementation/sqlalchemy/backend.py | 6 - .../sqlalchemy/querybuilder/main.py | 31 ++ aiida/orm/nodes/data/array/bands.py | 132 +++++++++ aiida/restapi/translator/nodes/node.py | 4 +- .../migrations/test_migrations_many.py | 8 +- tests/orm/test_querybuilder.py | 28 +- 22 files changed, 246 insertions(+), 729 deletions(-) delete mode 100644 aiida/backends/djsite/queries.py delete mode 100644 aiida/backends/general/abstractqueries.py rename aiida/{manage/database/integrity/duplicate_uuid.py => backends/general/migrations/duplicate_uuids.py} (71%) delete mode 100644 aiida/backends/sqlalchemy/queries.py diff --git a/aiida/backends/djsite/db/migrations/0014_add_node_uuid_unique_constraint.py b/aiida/backends/djsite/db/migrations/0014_add_node_uuid_unique_constraint.py index 0261e5b12c..6b2b666d5c 100644 --- a/aiida/backends/djsite/db/migrations/0014_add_node_uuid_unique_constraint.py +++ b/aiida/backends/djsite/db/migrations/0014_add_node_uuid_unique_constraint.py @@ -27,7 +27,7 @@ def verify_node_uuid_uniqueness(_, __): :raises: IntegrityError if database contains nodes with duplicate UUIDS. """ - from aiida.backends.general.migrations.utils import verify_uuid_uniqueness + from aiida.backends.general.migrations.duplicate_uuids import verify_uuid_uniqueness verify_uuid_uniqueness(table='db_dbnode') diff --git a/aiida/backends/djsite/db/migrations/0018_django_1_11.py b/aiida/backends/djsite/db/migrations/0018_django_1_11.py index b096daffd5..64cb8c797f 100644 --- a/aiida/backends/djsite/db/migrations/0018_django_1_11.py +++ b/aiida/backends/djsite/db/migrations/0018_django_1_11.py @@ -33,7 +33,7 @@ def _verify_uuid_uniqueness(apps, schema_editor): :raises: IntegrityError if database contains rows with duplicate UUIDS. """ # pylint: disable=unused-argument - from aiida.manage.database.integrity.duplicate_uuid import verify_uuid_uniqueness + from aiida.backends.general.migrations.duplicate_uuids import verify_uuid_uniqueness for table in tables: verify_uuid_uniqueness(table=table) diff --git a/aiida/backends/djsite/queries.py b/aiida/backends/djsite/queries.py deleted file mode 100644 index 9bc27d4d92..0000000000 --- a/aiida/backends/djsite/queries.py +++ /dev/null @@ -1,229 +0,0 @@ -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### -"""Django query backend.""" - -# pylint: disable=import-error,no-name-in-module -from aiida.backends.general.abstractqueries import AbstractQueryManager - - -class DjangoQueryManager(AbstractQueryManager): - """Object that mananges the Django queries.""" - - def get_creation_statistics(self, user_pk=None): - """ - Return a dictionary with the statistics of node creation, summarized by day, - optimized for the Django backend. - - :note: Days when no nodes were created are not present in the returned `ctime_by_day` dictionary. - - :param user_pk: If None (default), return statistics for all users. - If user pk is specified, return only the statistics for the given user. - - :return: a dictionary as - follows:: - - { - "total": TOTAL_NUM_OF_NODES, - "types": {TYPESTRING1: count, TYPESTRING2: count, ...}, - "ctime_by_day": {'YYYY-MMM-DD': count, ...} - - where in `ctime_by_day` the key is a string in the format 'YYYY-MM-DD' and the value is - an integer with the number of nodes created that day.""" - # pylint: disable=no-member - import sqlalchemy as sa - import aiida.backends.djsite.db.models as djmodels - from aiida.manage.manager import get_manager - backend = get_manager().get_backend() - - # Get the session (uses internally aldjemy - so, sqlalchemy) also for the Djsite backend - session = backend.get_session() - - retdict = {} - - total_query = session.query(djmodels.DbNode.sa) - types_query = session.query( - djmodels.DbNode.sa.node_type.label('typestring'), sa.func.count(djmodels.DbNode.sa.id) - ) - stat_query = session.query( - sa.func.date_trunc('day', djmodels.DbNode.sa.ctime).label('cday'), sa.func.count(djmodels.DbNode.sa.id) - ) - - if user_pk is not None: - total_query = total_query.filter(djmodels.DbNode.sa.user_id == user_pk) - types_query = types_query.filter(djmodels.DbNode.sa.user_id == user_pk) - stat_query = stat_query.filter(djmodels.DbNode.sa.user_id == user_pk) - - # Total number of nodes - retdict['total'] = total_query.count() - - # Nodes per type - retdict['types'] = dict(types_query.group_by('typestring').all()) - - # Nodes created per day - stat = stat_query.group_by('cday').order_by('cday').all() - - ctime_by_day = {_[0].strftime('%Y-%m-%d'): _[1] for _ in stat} - retdict['ctime_by_day'] = ctime_by_day - - return retdict - # Still not containing all dates - # temporary fix only for DJANGO backend - # Will be useless when the _join_ancestors method of the QueryBuilder - # will be re-implemented without using the DbPath - - @staticmethod - def query_past_days(q_object, args): - """ - Subselect to filter data nodes by their age. - - :param q_object: a query object - :param args: a namespace with parsed command line parameters. - """ - from aiida.common import timezone - from django.db.models import Q - import datetime - if args.past_days is not None: - now = timezone.now() - n_days_ago = now - datetime.timedelta(days=args.past_days) - q_object.add(Q(ctime__gte=n_days_ago), Q.AND) - - @staticmethod - def query_group(q_object, args): - """ - Subselect to filter data nodes by their group. - - :param q_object: a query object - :param args: a namespace with parsed command line parameters. - """ - from django.db.models import Q - if args.group_name is not None: - q_object.add(Q(dbgroups__name__in=args.group_name), Q.AND) - if args.group_pk is not None: - q_object.add(Q(dbgroups__pk__in=args.group_pk), Q.AND) - - def get_bands_and_parents_structure(self, args): - """Returns bands and closest parent structure.""" - # pylint: disable=too-many-locals - from django.db.models import Q - from aiida.backends.djsite.db import models - from aiida.common.utils import grouper - from aiida.orm import BandsData - - q_object = None - if args.all_users is False: - from aiida import orm - q_object = Q(user__id=orm.User.objects.get_default().id) - else: - q_object = Q() - - self.query_past_days(q_object, args) - self.query_group(q_object, args) - - bands_list_data = models.DbNode.objects.filter( - node_type__startswith=BandsData.class_node_type - ).filter(q_object).distinct().order_by('ctime').values_list('pk', 'label', 'ctime') - - entry_list = [] - # the frist argument of the grouper function is the query group size. - for this_chunk in grouper(100, [(_[0], _[1], _[2]) for _ in bands_list_data]): - # gather all banddata pks - pks = [_[0] for _ in this_chunk] - - # get the closest structures (WITHOUT DbPath) - structure_dict = get_closest_parents(pks, Q(node_type='data.core.structure.StructureData.'), chunk_size=1) - - struc_pks = [structure_dict.get(pk) for pk in pks] - - # query for the attributes needed for the structure formula - res_attr = models.DbNode.objects.filter(id__in=struc_pks).values_list('id', 'attributes') - res_attr = {rattr[0]: rattr[1] for rattr in res_attr} - - # prepare the printout - for (b_id_lbl_date, struc_pk) in zip(this_chunk, struc_pks): - if struc_pk is not None: - strct = res_attr[struc_pk] - akinds, asites = strct['kinds'], strct['sites'] - formula = self._extract_formula(akinds, asites, args) - else: - if args.element is not None or args.element_only is not None: - formula = None - else: - formula = '<>' - - if formula is None: - continue - entry_list.append([ - str(b_id_lbl_date[0]), - str(formula), b_id_lbl_date[2].strftime('%d %b %Y'), b_id_lbl_date[1] - ]) - - return entry_list - - -def get_closest_parents(pks, *args, **kwargs): - """Get the closest parents dbnodes of a set of nodes. - - :param pks: one pk or an iterable of pks of nodes - :param chunk_size: we chunk the pks into groups of this size, - to optimize the speed (default=50) - :param print_progress: print the the progression if True (default=False). - :param args: additional query parameters - :param kwargs: additional query parameters - :returns: a dictionary of the form - pk1: pk of closest parent of node with pk1, - pk2: pk of closest parent of node with pk2 - - .. note:: It works also if pks is a list of nodes rather than their pks - - .. todo:: find a way to always get a parent (when there is one) from each pk. - Now, when the same parent has several children in pks, only - one of them is kept. This is a BUG, related to the use of a dictionary - (children_dict, see below...). - For now a work around is to use chunk_size=1.""" - - from copy import deepcopy - from aiida.backends.djsite.db import models - from aiida.common.utils import grouper - - chunk_size = kwargs.pop('chunk_size', 50) - print_progress = kwargs.pop('print_progress', False) - - result_dict = {} - if print_progress: - print('Chunk size:', chunk_size) - - for i, chunk_pks in enumerate(grouper(chunk_size, list(set(pks)) if isinstance(pks, list) else [pks])): - if print_progress: - print('Dealing with chunk #', i) - result_chunk_dict = {} - - q_pks = models.DbNode.objects.filter(pk__in=chunk_pks).values_list('pk', flat=True) - # Now I am looking for parents (depth=0) of the nodes in the chunk: - - q_inputs = models.DbNode.objects.filter(outputs__pk__in=q_pks).distinct() - depth = -1 # to be consistent with the DbPath depth (=0 for direct inputs) - children_dict = {k: v for k, v in q_inputs.values_list('pk', 'outputs__pk') if v in q_pks} - # While I haven't found a closest ancestor for every member of chunk_pks: - while q_inputs.count() > 0 and len(result_chunk_dict) < len(chunk_pks): - depth += 1 - q_inp_filtered = q_inputs.filter(*args, **kwargs) - if q_inp_filtered.count() > 0: - result_chunk_dict.update({(children_dict[k], k) - for k in q_inp_filtered.values_list('pk', flat=True) - if children_dict[k] not in result_chunk_dict}) - inputs = list(q_inputs.values_list('pk', flat=True)) - q_inputs = models.DbNode.objects.filter(outputs__pk__in=inputs).distinct() - - q_inputs_dict = {k: children_dict[v] for k, v in q_inputs.values_list('pk', 'outputs__pk') if v in inputs} - children_dict = deepcopy(q_inputs_dict) - - result_dict.update(result_chunk_dict) - - return result_dict diff --git a/aiida/backends/general/abstractqueries.py b/aiida/backends/general/abstractqueries.py deleted file mode 100644 index 0cc911879f..0000000000 --- a/aiida/backends/general/abstractqueries.py +++ /dev/null @@ -1,265 +0,0 @@ -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### -"""Manage AiiDA queries.""" -import abc - - -class AbstractQueryManager(abc.ABC): - """Manage AiiDA queries.""" - - def __init__(self, backend): - """ - :param backend: The AiiDA backend - :type backend: :class:`aiida.orm.implementation.sql.backends.SqlBackend` - """ - self._backend = backend - - def get_duplicate_uuids(self, table): - """ - Return a list of rows with identical UUID - - :param table: Database table with uuid column, e.g. 'db_dbnode' - :type str: - - :return: list of tuples of (id, uuid) of rows with duplicate UUIDs - :rtype list: - """ - query = f""" - SELECT s.id, s.uuid FROM (SELECT *, COUNT(*) OVER(PARTITION BY uuid) AS c FROM {table}) - AS s WHERE c > 1 - """ - return self._backend.execute_raw(query) - - def apply_new_uuid_mapping(self, table, mapping): - for pk, uuid in mapping.items(): - query = f"""UPDATE {table} SET uuid = '{uuid}' WHERE id = {pk}""" - with self._backend.cursor() as cursor: - cursor.execute(query) - - @staticmethod - def get_creation_statistics(user_pk=None): - """ - Return a dictionary with the statistics of node creation, summarized by day. - - :note: Days when no nodes were created are not present in the returned `ctime_by_day` dictionary. - - :param user_pk: If None (default), return statistics for all users. - If user pk is specified, return only the statistics for the given user. - - :return: a dictionary as - follows:: - - { - "total": TOTAL_NUM_OF_NODES, - "types": {TYPESTRING1: count, TYPESTRING2: count, ...}, - "ctime_by_day": {'YYYY-MMM-DD': count, ...} - - where in `ctime_by_day` the key is a string in the format 'YYYY-MM-DD' and the value is - an integer with the number of nodes created that day. - """ - import datetime - from collections import Counter - from aiida.orm import User, Node, QueryBuilder - - def count_statistics(dataset): - - def get_statistics_dict(dataset): - results = {} - for count, typestring in sorted((v, k) for k, v in dataset.items())[::-1]: - results[typestring] = count - return results - - count_dict = {} - - types = Counter([r[2] for r in dataset]) - count_dict['types'] = get_statistics_dict(types) - - ctimelist = [r[1].strftime('%Y-%m-%d') for r in dataset] - ctime = Counter(ctimelist) - - if ctimelist: - - # For the way the string is formatted, we can just sort it alphabetically - firstdate = datetime.datetime.strptime(sorted(ctimelist)[0], '%Y-%m-%d') - lastdate = datetime.datetime.strptime(sorted(ctimelist)[-1], '%Y-%m-%d') - - curdate = firstdate - outdata = {} - - while curdate <= lastdate: - curdatestring = curdate.strftime('%Y-%m-%d') - outdata[curdatestring] = ctime.get(curdatestring, 0) - curdate += datetime.timedelta(days=1) - count_dict['ctime_by_day'] = outdata - - else: - count_dict['ctime_by_day'] = {} - - return count_dict - - statistics = {} - - q_build = QueryBuilder() - q_build.append(Node, project=['id', 'ctime', 'type'], tag='node') - - if user_pk is not None: - q_build.append(User, with_node='node', project='email', filters={'pk': user_pk}) - qb_res = q_build.all() - - # total count - statistics['total'] = len(qb_res) - statistics.update(count_statistics(qb_res)) - - return statistics - - @staticmethod - def _extract_formula(akinds, asites, args): - """ - Extract formula from the structure object. - - :param akinds: list of kinds, e.g. [{'mass': 55.845, 'name': 'Fe', 'symbols': ['Fe'], 'weights': [1.0]}, - {'mass': 15.9994, 'name': 'O', 'symbols': ['O'], 'weights': [1.0]}] - :param asites: list of structure sites e.g. [{'position': [0.0, 0.0, 0.0], 'kind_name': 'Fe'}, - {'position': [2.0, 2.0, 2.0], 'kind_name': 'O'}] - :param args: a namespace with parsed command line parameters, here only 'element' and 'element_only' are used - :type args: dict - - :return: a string with formula if the formula is found - """ - from aiida.orm.nodes.data.structure import (get_formula, get_symbols_string) - - if args.element is not None: - all_symbols = [_['symbols'][0] for _ in akinds] - if not any(s in args.element for s in all_symbols): - return None - - if args.element_only is not None: - all_symbols = [_['symbols'][0] for _ in akinds] - if not all(s in all_symbols for s in args.element_only): - return None - - # We want only the StructureData that have attributes - if akinds is None or asites is None: - return '<>' - - symbol_dict = {} - for k in akinds: - symbols = k['symbols'] - weights = k['weights'] - symbol_dict[k['name']] = get_symbols_string(symbols, weights) - - try: - symbol_list = [] - for site in asites: - symbol_list.append(symbol_dict[site['kind_name']]) - formula = get_formula(symbol_list, mode=args.formulamode) - # If for some reason there is no kind with the name - # referenced by the site - except KeyError: - formula = '<>' - return formula - - def get_bands_and_parents_structure(self, args): - """Search for bands and return bands and the closest structure that is a parent of the instance. - This is the backend independent way, can be overriden for performance reason - - :returns: - A list of sublists, each latter containing (in order): - pk as string, formula as string, creation date, bandsdata-label - """ - # pylint: disable=too-many-locals - - import datetime - from aiida.common import timezone - from aiida import orm - - q_build = orm.QueryBuilder() - if args.all_users is False: - q_build.append(orm.User, tag='creator', filters={'email': orm.User.objects.get_default().email}) - else: - q_build.append(orm.User, tag='creator') - - group_filters = {} - - if args.group_name is not None: - group_filters.update({'name': {'in': args.group_name}}) - if args.group_pk is not None: - group_filters.update({'id': {'in': args.group_pk}}) - - q_build.append(orm.Group, tag='group', filters=group_filters, with_user='creator') - - bdata_filters = {} - if args.past_days is not None: - bdata_filters.update({'ctime': {'>=': timezone.now() - datetime.timedelta(days=args.past_days)}}) - - q_build.append( - orm.BandsData, tag='bdata', with_group='group', filters=bdata_filters, project=['id', 'label', 'ctime'] - ) - bands_list_data = q_build.all() - - q_build.append( - orm.StructureData, - tag='sdata', - with_descendants='bdata', - # We don't care about the creator of StructureData - project=['id', 'attributes.kinds', 'attributes.sites'] - ) - - q_build.order_by({orm.StructureData: {'ctime': 'desc'}}) - - structure_dict = dict() - list_data = q_build.distinct().all() - for bid, _, _, _, akinds, asites in list_data: - structure_dict[bid] = (akinds, asites) - - entry_list = [] - already_visited_bdata = set() - - for [bid, blabel, bdate] in bands_list_data: - - # We process only one StructureData per BandsData. - # We want to process the closest StructureData to - # every BandsData. - # We hope that the StructureData with the latest - # creation time is the closest one. - # This will be updated when the QueryBuilder supports - # order_by by the distance of two nodes. - if already_visited_bdata.__contains__(bid): - continue - already_visited_bdata.add(bid) - strct = structure_dict.get(bid, None) - - if strct is not None: - akinds, asites = strct - formula = self._extract_formula(akinds, asites, args) - else: - if args.element is not None or args.element_only is not None: - formula = None - else: - formula = '<>' - - if formula is None: - continue - entry_list.append([str(bid), str(formula), bdate.strftime('%d %b %Y'), blabel]) - - return entry_list - - @staticmethod - def get_all_parents(node_pks, return_values=('id',)): - """Get all the parents of given nodes - - :param node_pks: one node pk or an iterable of node pks - :return: a list of aiida objects with all the parents of the nodes""" - from aiida.orm import Node, QueryBuilder - - q_build = QueryBuilder() - q_build.append(Node, tag='low_node', filters={'id': {'in': node_pks}}) - q_build.append(Node, with_descendants='low_node', project=return_values) - return q_build.all() diff --git a/aiida/manage/database/integrity/duplicate_uuid.py b/aiida/backends/general/migrations/duplicate_uuids.py similarity index 71% rename from aiida/manage/database/integrity/duplicate_uuid.py rename to aiida/backends/general/migrations/duplicate_uuids.py index 503efd3ebb..e6f4ea59bc 100644 --- a/aiida/manage/database/integrity/duplicate_uuid.py +++ b/aiida/backends/general/migrations/duplicate_uuids.py @@ -8,23 +8,26 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Generic functions to verify the integrity of the database and optionally apply patches to fix problems.""" +import os from aiida.common import exceptions -from aiida.manage.manager import get_manager -__all__ = ('verify_uuid_uniqueness', 'get_duplicate_uuids', 'deduplicate_uuids', 'TABLES_UUID_DEDUPLICATION') +TABLES_UUID_DEDUPLICATION = ('db_dbcomment', 'db_dbcomputer', 'db_dbgroup', 'db_dbnode') -TABLES_UUID_DEDUPLICATION = ['db_dbcomment', 'db_dbcomputer', 'db_dbgroup', 'db_dbnode'] - -def get_duplicate_uuids(table): +def _get_duplicate_uuids(table): """Retrieve rows with duplicate UUIDS. :param table: database table with uuid column, e.g. 'db_dbnode' :return: list of tuples of (id, uuid) of rows with duplicate UUIDs """ + from aiida.manage.manager import get_manager backend = get_manager().get_backend() - return backend.query_manager.get_duplicate_uuids(table=table) + query = f""" + SELECT s.id, s.uuid FROM (SELECT *, COUNT(*) OVER(PARTITION BY uuid) AS c FROM {table}) + AS s WHERE c > 1 + """ + return backend.execute_raw(query) def verify_uuid_uniqueness(table): @@ -35,7 +38,7 @@ def verify_uuid_uniqueness(table): :raises: IntegrityError if table contains rows with duplicate UUIDS. """ - duplicates = get_duplicate_uuids(table=table) + duplicates = _get_duplicate_uuids(table=table) if duplicates: raise exceptions.IntegrityError( @@ -44,14 +47,18 @@ def verify_uuid_uniqueness(table): ) -def apply_new_uuid_mapping(table, mapping): +def _apply_new_uuid_mapping(table, mapping): """Take a mapping of pks to UUIDs and apply it to the given table. :param table: database table with uuid column, e.g. 'db_dbnode' :param mapping: dictionary of UUIDs mapped onto a pk """ + from aiida.manage.manager import get_manager backend = get_manager().get_backend() - backend.query_manager.apply_new_uuid_mapping(table, mapping) + for pk, uuid in mapping.items(): + query = f"""UPDATE {table} SET uuid = '{uuid}' WHERE id = {pk}""" + with backend.cursor() as cursor: + cursor.execute(query) def deduplicate_uuids(table=None, dry_run=True): @@ -64,20 +71,21 @@ def deduplicate_uuids(table=None, dry_run=True): duplicate UUIDs in an inconsistent state. This command will run an analysis to detect duplicate UUIDs in a given table and solve it by generating new UUIDs. Note that it will not delete or merge any rows. - :param dry_run: when True, no actual changes will be made - :return: list of strings denoting the performed operations, or those that would have been applied for dry_run=False + :return: list of strings denoting the performed operations :raises ValueError: if the specified table is invalid """ + import distutils.dir_util from collections import defaultdict from aiida.common.utils import get_new_uuid + from .utils import get_node_repository_sub_folder if table not in TABLES_UUID_DEDUPLICATION: raise ValueError(f"invalid table {table}: choose from {', '.join(TABLES_UUID_DEDUPLICATION)}") mapping = defaultdict(list) - for pk, uuid in get_duplicate_uuids(table=table): + for pk, uuid in _get_duplicate_uuids(table=table): mapping[uuid].append(int(pk)) messages = [] @@ -101,9 +109,16 @@ def deduplicate_uuids(table=None, dry_run=True): messages.append(f'would update UUID of {table} row<{pk}> from {uuid_ref} to {uuid_new}') else: messages.append(f'updated UUID of {table} row<{pk}> from {uuid_ref} to {uuid_new}') + dirpath_repo_ref = get_node_repository_sub_folder(uuid_ref) + dirpath_repo_new = get_node_repository_sub_folder(uuid_new) + + # First make sure the new repository exists, then copy the contents of the ref into the new. We use the + # somewhat unknown `distuitils.dir_util` method since that does just contents as we want. + os.makedirs(dirpath_repo_new, exist_ok=True) + distutils.dir_util.copy_tree(dirpath_repo_ref, dirpath_repo_new) if not dry_run: - apply_new_uuid_mapping(table, mapping_new_uuid) + _apply_new_uuid_mapping(table, mapping_new_uuid) if not messages: messages = ['no duplicate UUIDs found'] diff --git a/aiida/backends/general/migrations/utils.py b/aiida/backends/general/migrations/utils.py index b50f6de180..63cd9dad4b 100644 --- a/aiida/backends/general/migrations/utils.py +++ b/aiida/backends/general/migrations/utils.py @@ -435,99 +435,3 @@ def recursive_datetime_to_isoformat(value): def dumps_json(dictionary): """Transforms all datetime object into isoformat and then returns the JSON.""" return json.dumps(recursive_datetime_to_isoformat(dictionary)) - - -def get_duplicate_uuids(table): - """Retrieve rows with duplicate UUIDS. - - :param table: database table with uuid column, e.g. 'db_dbnode' - :return: list of tuples of (id, uuid) of rows with duplicate UUIDs - """ - from aiida.manage.manager import get_manager - backend = get_manager().get_backend() - return backend.query_manager.get_duplicate_uuids(table=table) - - -def verify_uuid_uniqueness(table): - """Check whether database table contains rows with duplicate UUIDS. - - :param table: Database table with uuid column, e.g. 'db_dbnode' - :type str: - - :raises: IntegrityError if table contains rows with duplicate UUIDS. - """ - duplicates = get_duplicate_uuids(table=table) - - if duplicates: - raise exceptions.IntegrityError( - 'Table {table:} contains rows with duplicate UUIDS: run ' - '`verdi database integrity detect-duplicate-uuid -t {table:}` to address the problem'.format(table=table) - ) - - -def apply_new_uuid_mapping(table, mapping): - """Take a mapping of pks to UUIDs and apply it to the given table. - - :param table: database table with uuid column, e.g. 'db_dbnode' - :param mapping: dictionary of UUIDs mapped onto a pk - """ - from aiida.manage.manager import get_manager - backend = get_manager().get_backend() - backend.query_manager.apply_new_uuid_mapping(table, mapping) - - -def deduplicate_uuids(table=None): - """Detect and solve entities with duplicate UUIDs in a given database table. - - Before aiida-core v1.0.0, there was no uniqueness constraint on the UUID column of the node table in the database - and a few other tables as well. This made it possible to store multiple entities with identical UUIDs in the same - table without the database complaining. This bug was fixed in aiida-core=1.0.0 by putting an explicit uniqueness - constraint on UUIDs on the database level. However, this would leave databases created before this patch with - duplicate UUIDs in an inconsistent state. This command will run an analysis to detect duplicate UUIDs in a given - table and solve it by generating new UUIDs. Note that it will not delete or merge any rows. - - :return: list of strings denoting the performed operations - :raises ValueError: if the specified table is invalid - """ - import distutils.dir_util - from collections import defaultdict - - from aiida.common.utils import get_new_uuid - - mapping = defaultdict(list) - - for pk, uuid in get_duplicate_uuids(table=table): - mapping[uuid].append(int(pk)) - - messages = [] - mapping_new_uuid = {} - - for uuid, rows in mapping.items(): - - uuid_ref = None - - for pk in rows: - - # We don't have to change all rows that have the same UUID, the first one can keep the original - if uuid_ref is None: - uuid_ref = uuid - continue - - uuid_new = str(get_new_uuid()) - mapping_new_uuid[pk] = uuid_new - - messages.append(f'updated UUID of {table} row<{pk}> from {uuid_ref} to {uuid_new}') - dirpath_repo_ref = get_node_repository_sub_folder(uuid_ref) - dirpath_repo_new = get_node_repository_sub_folder(uuid_new) - - # First make sure the new repository exists, then copy the contents of the ref into the new. We use the - # somewhat unknown `distuitils.dir_util` method since that does just contents as we want. - os.makedirs(dirpath_repo_new, exist_ok=True) - distutils.dir_util.copy_tree(dirpath_repo_ref, dirpath_repo_new) - - apply_new_uuid_mapping(table, mapping_new_uuid) - - if not messages: - messages = ['no duplicate UUIDs found'] - - return messages diff --git a/aiida/backends/sqlalchemy/migrations/versions/37f3d4882837_make_all_uuid_columns_unique.py b/aiida/backends/sqlalchemy/migrations/versions/37f3d4882837_make_all_uuid_columns_unique.py index 8974df9d88..7d13b730cd 100644 --- a/aiida/backends/sqlalchemy/migrations/versions/37f3d4882837_make_all_uuid_columns_unique.py +++ b/aiida/backends/sqlalchemy/migrations/versions/37f3d4882837_make_all_uuid_columns_unique.py @@ -34,7 +34,8 @@ def verify_uuid_uniqueness(table): """Check whether the database contains duplicate UUIDS. - Note that we have to redefine this method from aiida.manage.database.integrity.verify_uuid_uniqueness + Note that we have to redefine this method from + aiida.backends.general.migrations.duplicate_uuids.verify_uuid_uniqueness because that uses the default database connection, while here the one created by Alembic should be used instead. :raises: IntegrityError if database contains nodes with duplicate UUIDS. diff --git a/aiida/backends/sqlalchemy/queries.py b/aiida/backends/sqlalchemy/queries.py deleted file mode 100644 index 4e1d9409a9..0000000000 --- a/aiida/backends/sqlalchemy/queries.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### -"""Module to manage custom queries under SQLA backend.""" -from aiida.backends.general.abstractqueries import AbstractQueryManager - - -class SqlaQueryManager(AbstractQueryManager): - """SQLAlchemy implementation of custom queries, for efficiency reasons.""" - - def get_creation_statistics(self, user_pk=None): - """Return a dictionary with the statistics of node creation, summarized by day, - optimized for the Django backend. - - :note: Days when no nodes were created are not present in the returned `ctime_by_day` dictionary. - - :param user_pk: If None (default), return statistics for all users. - If user pk is specified, return only the statistics for the given user. - - :return: a dictionary as - follows:: - - { - "total": TOTAL_NUM_OF_NODES, - "types": {TYPESTRING1: count, TYPESTRING2: count, ...}, - "ctime_by_day": {'YYYY-MMM-DD': count, ...} - - where in `ctime_by_day` the key is a string in the format 'YYYY-MM-DD' and the value is - an integer with the number of nodes created that day.""" - import sqlalchemy as sa - import aiida.backends.sqlalchemy - from aiida.backends.sqlalchemy import models as m - - # Get the session (uses internally aldjemy - so, sqlalchemy) also for the Djsite backend - session = aiida.backends.sqlalchemy.get_scoped_session() - - retdict = {} - - total_query = session.query(m.node.DbNode) - types_query = session.query(m.node.DbNode.node_type.label('typestring'), sa.func.count(m.node.DbNode.id)) # pylint: disable=no-member - stat_query = session.query( - sa.func.date_trunc('day', m.node.DbNode.ctime).label('cday'), # pylint: disable=no-member - sa.func.count(m.node.DbNode.id) # pylint: disable=no-member - ) - - if user_pk is not None: - total_query = total_query.filter(m.node.DbNode.user_id == user_pk) - types_query = types_query.filter(m.node.DbNode.user_id == user_pk) - stat_query = stat_query.filter(m.node.DbNode.user_id == user_pk) - - # Total number of nodes - retdict['total'] = total_query.count() - - # Nodes per type - retdict['types'] = dict(types_query.group_by('typestring').all()) - - # Nodes created per day - stat = stat_query.group_by('cday').order_by('cday').all() - - ctime_by_day = {_[0].strftime('%Y-%m-%d'): _[1] for _ in stat} - retdict['ctime_by_day'] = ctime_by_day - - return retdict - # Still not containing all dates diff --git a/aiida/cmdline/commands/cmd_data/cmd_bands.py b/aiida/cmdline/commands/cmd_data/cmd_bands.py index 5ce5b55e91..eefb497492 100644 --- a/aiida/cmdline/commands/cmd_data/cmd_bands.py +++ b/aiida/cmdline/commands/cmd_data/cmd_bands.py @@ -41,11 +41,10 @@ def bands(): @options.FORMULA_MODE() def bands_list(elements, elements_exclusive, raw, formula_mode, past_days, groups, all_users): """List BandsData objects.""" - from aiida.manage.manager import get_manager from tabulate import tabulate from argparse import Namespace - backend = get_manager().get_backend() + from aiida.orm.nodes.data.array.bands import get_bands_and_parents_structure args = Namespace() args.element = elements @@ -59,8 +58,7 @@ def bands_list(elements, elements_exclusive, raw, formula_mode, past_days, group args.group_pk = None args.all_users = all_users - query = backend.query_manager - entry_list = query.get_bands_and_parents_structure(args) + entry_list = get_bands_and_parents_structure(args) counter = 0 bands_list_data = list() diff --git a/aiida/cmdline/commands/cmd_database.py b/aiida/cmdline/commands/cmd_database.py index c70aeba485..5e4b93b5ba 100644 --- a/aiida/cmdline/commands/cmd_database.py +++ b/aiida/cmdline/commands/cmd_database.py @@ -15,7 +15,7 @@ from aiida.cmdline.commands.cmd_verdi import verdi from aiida.cmdline.params import options from aiida.cmdline.utils import decorators, echo -from aiida.manage.database.integrity.duplicate_uuid import TABLES_UUID_DEDUPLICATION +from aiida.backends.general.migrations.duplicate_uuids import TABLES_UUID_DEDUPLICATION @verdi.group('database') @@ -120,7 +120,7 @@ def detect_duplicate_uuid(table, apply_patch): duplicate UUIDs in an inconsistent state. This command will run an analysis to detect duplicate UUIDs in a given table and solve it by generating new UUIDs. Note that it will not delete or merge any rows. """ - from aiida.manage.database.integrity.duplicate_uuid import deduplicate_uuids + from aiida.backends.general.migrations.duplicate_uuids import deduplicate_uuids from aiida.manage.manager import get_manager manager = get_manager() diff --git a/aiida/manage/__init__.py b/aiida/manage/__init__.py index bcb5001a50..55552c58b1 100644 --- a/aiida/manage/__init__.py +++ b/aiida/manage/__init__.py @@ -46,22 +46,18 @@ 'ProcessLauncher', 'Profile', 'RemoteException', - 'TABLES_UUID_DEDUPLICATION', 'check_and_migrate_config', 'config_needs_migrating', 'config_schema', - 'deduplicate_uuids', 'disable_caching', 'enable_caching', 'get_current_version', - 'get_duplicate_uuids', 'get_manager', 'get_option', 'get_option_names', 'get_use_cache', 'parse_option', 'reset_manager', - 'verify_uuid_uniqueness', 'write_database_integrity_violation', ) diff --git a/aiida/manage/database/__init__.py b/aiida/manage/database/__init__.py index f1222d7e8b..9936f125fe 100644 --- a/aiida/manage/database/__init__.py +++ b/aiida/manage/database/__init__.py @@ -17,10 +17,6 @@ from .integrity import * __all__ = ( - 'TABLES_UUID_DEDUPLICATION', - 'deduplicate_uuids', - 'get_duplicate_uuids', - 'verify_uuid_uniqueness', 'write_database_integrity_violation', ) diff --git a/aiida/manage/database/integrity/__init__.py b/aiida/manage/database/integrity/__init__.py index 6bb9ca7f26..01c64cc9db 100644 --- a/aiida/manage/database/integrity/__init__.py +++ b/aiida/manage/database/integrity/__init__.py @@ -14,14 +14,9 @@ # yapf: disable # pylint: disable=wildcard-import -from .duplicate_uuid import * from .utils import * __all__ = ( - 'TABLES_UUID_DEDUPLICATION', - 'deduplicate_uuids', - 'get_duplicate_uuids', - 'verify_uuid_uniqueness', 'write_database_integrity_violation', ) diff --git a/aiida/orm/implementation/backends.py b/aiida/orm/implementation/backends.py index 701f4a2aba..0308ebf911 100644 --- a/aiida/orm/implementation/backends.py +++ b/aiida/orm/implementation/backends.py @@ -18,7 +18,6 @@ BackendAuthInfoCollection, BackendCommentCollection, BackendComputerCollection, BackendGroupCollection, BackendLogCollection, BackendNodeCollection, BackendQueryBuilder, BackendUserCollection ) - from aiida.backends.general.abstractqueries import AbstractQueryManager __all__ = ('Backend',) @@ -60,11 +59,6 @@ def logs(self) -> 'BackendLogCollection': def nodes(self) -> 'BackendNodeCollection': """Return the collection of nodes""" - @property - @abc.abstractmethod - def query_manager(self) -> 'AbstractQueryManager': - """Return the query manager for the objects stored in the backend""" - @abc.abstractmethod def query(self) -> 'BackendQueryBuilder': """Return an instance of a query builder implementation for this backend""" diff --git a/aiida/orm/implementation/django/backend.py b/aiida/orm/implementation/django/backend.py index 6de13e3f02..ba4799a3ba 100644 --- a/aiida/orm/implementation/django/backend.py +++ b/aiida/orm/implementation/django/backend.py @@ -13,7 +13,6 @@ # pylint: disable=import-error,no-name-in-module from django.db import models, transaction -from aiida.backends.djsite.queries import DjangoQueryManager from aiida.backends.djsite.manager import DjangoBackendManager from ..sql.backends import SqlBackend @@ -41,7 +40,6 @@ def __init__(self): self._groups = groups.DjangoGroupCollection(self) self._logs = logs.DjangoLogCollection(self) self._nodes = nodes.DjangoNodeCollection(self) - self._query_manager = DjangoQueryManager(self) self._backend_manager = DjangoBackendManager() self._users = users.DjangoUserCollection(self) @@ -72,10 +70,6 @@ def logs(self): def nodes(self): return self._nodes - @property - def query_manager(self): - return self._query_manager - def query(self): return querybuilder.DjangoQueryBuilder(self) diff --git a/aiida/orm/implementation/querybuilder.py b/aiida/orm/implementation/querybuilder.py index 0eb05856d1..9f4c6be9eb 100644 --- a/aiida/orm/implementation/querybuilder.py +++ b/aiida/orm/implementation/querybuilder.py @@ -94,7 +94,7 @@ class QueryDictType(TypedDict): GROUP_ENTITY_TYPE_PREFIX = 'group.' -class BackendQueryBuilder: +class BackendQueryBuilder(abc.ABC): """Backend query builder interface""" def __init__(self, backend: 'Backend'): @@ -155,3 +155,24 @@ def analyze_query(self, data: QueryDictType, execute: bool = True, verbose: bool :params verbose: Display additional information regarding the plan. """ raise NotImplementedError + + @abc.abstractmethod + def get_creation_statistics(self, user_pk: Optional[int] = None) -> Dict[str, Any]: + """Return a dictionary with the statistics of node creation, summarized by day. + + :note: Days when no nodes were created are not present in the returned `ctime_by_day` dictionary. + + :param user_pk: If None (default), return statistics for all users. + If user pk is specified, return only the statistics for the given user. + + :return: a dictionary as follows:: + + { + "total": TOTAL_NUM_OF_NODES, + "types": {TYPESTRING1: count, TYPESTRING2: count, ...}, + "ctime_by_day": {'YYYY-MMM-DD': count, ...} + } + + where in `ctime_by_day` the key is a string in the format 'YYYY-MM-DD' and the value is + an integer with the number of nodes created that day. + """ diff --git a/aiida/orm/implementation/sqlalchemy/backend.py b/aiida/orm/implementation/sqlalchemy/backend.py index fa4ba06941..3661ee44a7 100644 --- a/aiida/orm/implementation/sqlalchemy/backend.py +++ b/aiida/orm/implementation/sqlalchemy/backend.py @@ -11,7 +11,6 @@ from contextlib import contextmanager from aiida.backends.sqlalchemy.models import base -from aiida.backends.sqlalchemy.queries import SqlaQueryManager from aiida.backends.sqlalchemy.manager import SqlaBackendManager from ..sql.backends import SqlBackend @@ -39,7 +38,6 @@ def __init__(self): self._groups = groups.SqlaGroupCollection(self) self._logs = logs.SqlaLogCollection(self) self._nodes = nodes.SqlaNodeCollection(self) - self._query_manager = SqlaQueryManager(self) self._schema_manager = SqlaBackendManager() self._users = users.SqlaUserCollection(self) @@ -70,10 +68,6 @@ def logs(self): def nodes(self): return self._nodes - @property - def query_manager(self): - return self._query_manager - def query(self): return querybuilder.SqlaQueryBuilder(self) diff --git a/aiida/orm/implementation/sqlalchemy/querybuilder/main.py b/aiida/orm/implementation/sqlalchemy/querybuilder/main.py index f44d7cbd98..397598dba0 100644 --- a/aiida/orm/implementation/sqlalchemy/querybuilder/main.py +++ b/aiida/orm/implementation/sqlalchemy/querybuilder/main.py @@ -1072,3 +1072,34 @@ def analyze_query(self, data: QueryDictType, execute: bool = True, verbose: bool options = f' ({options})' if options else '' rows = self.get_session().execute(f'EXPLAIN{options} {compiled.string}').fetchall() return '\n'.join(row.values()[0] for row in rows) + + def get_creation_statistics(self, user_pk: Optional[int] = None) -> Dict[str, Any]: + session = self.get_session() + retdict = {} + + total_query = session.query(self.Node) + types_query = session.query(self.Node.node_type.label('typestring'), sa_func.count(self.Node.id)) # pylint: disable=no-member + stat_query = session.query( + sa_func.date_trunc('day', self.Node.ctime).label('cday'), # pylint: disable=no-member + sa_func.count(self.Node.id) # pylint: disable=no-member + ) + + if user_pk is not None: + total_query = total_query.filter(self.Node.user_id == user_pk) + types_query = types_query.filter(self.Node.user_id == user_pk) + stat_query = stat_query.filter(self.Node.user_id == user_pk) + + # Total number of nodes + retdict['total'] = total_query.count() + + # Nodes per type + retdict['types'] = dict(types_query.group_by('typestring').all()) + + # Nodes created per day + stat = stat_query.group_by('cday').order_by('cday').all() + + ctime_by_day = {_[0].strftime('%Y-%m-%d'): _[1] for _ in stat} + retdict['ctime_by_day'] = ctime_by_day + + return retdict + # Still not containing all dates diff --git a/aiida/orm/nodes/data/array/bands.py b/aiida/orm/nodes/data/array/bands.py index 84d84b7bed..426acc5302 100644 --- a/aiida/orm/nodes/data/array/bands.py +++ b/aiida/orm/nodes/data/array/bands.py @@ -1799,3 +1799,135 @@ def _prepare_json(self, main_file_name='', comments=True): # pylint: disable=un MATPLOTLIB_FOOTER_TEMPLATE_EXPORTFILE = Template("""pl.savefig("$fname", format="$format")""") MATPLOTLIB_FOOTER_TEMPLATE_EXPORTFILE_WITH_DPI = Template("""pl.savefig("$fname", format="$format", dpi=$dpi)""") + + +def get_bands_and_parents_structure(args): + """Search for bands and return bands and the closest structure that is a parent of the instance. + + :returns: + A list of sublists, each latter containing (in order): + pk as string, formula as string, creation date, bandsdata-label + """ + # pylint: disable=too-many-locals + + import datetime + from aiida.common import timezone + from aiida import orm + + q_build = orm.QueryBuilder() + if args.all_users is False: + q_build.append(orm.User, tag='creator', filters={'email': orm.User.objects.get_default().email}) + else: + q_build.append(orm.User, tag='creator') + + group_filters = {} + + if args.group_name is not None: + group_filters.update({'name': {'in': args.group_name}}) + if args.group_pk is not None: + group_filters.update({'id': {'in': args.group_pk}}) + + q_build.append(orm.Group, tag='group', filters=group_filters, with_user='creator') + + bdata_filters = {} + if args.past_days is not None: + bdata_filters.update({'ctime': {'>=': timezone.now() - datetime.timedelta(days=args.past_days)}}) + + q_build.append( + orm.BandsData, tag='bdata', with_group='group', filters=bdata_filters, project=['id', 'label', 'ctime'] + ) + bands_list_data = q_build.all() + + q_build.append( + orm.StructureData, + tag='sdata', + with_descendants='bdata', + # We don't care about the creator of StructureData + project=['id', 'attributes.kinds', 'attributes.sites'] + ) + + q_build.order_by({orm.StructureData: {'ctime': 'desc'}}) + + structure_dict = dict() + list_data = q_build.distinct().all() + for bid, _, _, _, akinds, asites in list_data: + structure_dict[bid] = (akinds, asites) + + entry_list = [] + already_visited_bdata = set() + + for [bid, blabel, bdate] in bands_list_data: + + # We process only one StructureData per BandsData. + # We want to process the closest StructureData to + # every BandsData. + # We hope that the StructureData with the latest + # creation time is the closest one. + # This will be updated when the QueryBuilder supports + # order_by by the distance of two nodes. + if already_visited_bdata.__contains__(bid): + continue + already_visited_bdata.add(bid) + strct = structure_dict.get(bid, None) + + if strct is not None: + akinds, asites = strct + formula = _extract_formula(akinds, asites, args) + else: + if args.element is not None or args.element_only is not None: + formula = None + else: + formula = '<>' + + if formula is None: + continue + entry_list.append([str(bid), str(formula), bdate.strftime('%d %b %Y'), blabel]) + + return entry_list + + +def _extract_formula(akinds, asites, args): + """ + Extract formula from the structure object. + + :param akinds: list of kinds, e.g. [{'mass': 55.845, 'name': 'Fe', 'symbols': ['Fe'], 'weights': [1.0]}, + {'mass': 15.9994, 'name': 'O', 'symbols': ['O'], 'weights': [1.0]}] + :param asites: list of structure sites e.g. [{'position': [0.0, 0.0, 0.0], 'kind_name': 'Fe'}, + {'position': [2.0, 2.0, 2.0], 'kind_name': 'O'}] + :param args: a namespace with parsed command line parameters, here only 'element' and 'element_only' are used + :type args: dict + + :return: a string with formula if the formula is found + """ + from aiida.orm.nodes.data.structure import (get_formula, get_symbols_string) + + if args.element is not None: + all_symbols = [_['symbols'][0] for _ in akinds] + if not any(s in args.element for s in all_symbols): + return None + + if args.element_only is not None: + all_symbols = [_['symbols'][0] for _ in akinds] + if not all(s in all_symbols for s in args.element_only): + return None + + # We want only the StructureData that have attributes + if akinds is None or asites is None: + return '<>' + + symbol_dict = {} + for k in akinds: + symbols = k['symbols'] + weights = k['weights'] + symbol_dict[k['name']] = get_symbols_string(symbols, weights) + + try: + symbol_list = [] + for site in asites: + symbol_list.append(symbol_dict[site['kind_name']]) + formula = get_formula(symbol_list, mode=args.formulamode) + # If for some reason there is no kind with the name + # referenced by the site + except KeyError: + formula = '<>' + return formula diff --git a/aiida/restapi/translator/nodes/node.py b/aiida/restapi/translator/nodes/node.py index a5fa968151..5a4ce8b7e5 100644 --- a/aiida/restapi/translator/nodes/node.py +++ b/aiida/restapi/translator/nodes/node.py @@ -594,9 +594,7 @@ def get_formatted_result(self, label): def get_statistics(self, user_pk=None): """Return statistics for a given node""" - - qmanager = self._backend.query_manager - return qmanager.get_creation_statistics(user_pk=user_pk) + return self._backend.query().get_creation_statistics(user_pk=user_pk) @staticmethod def get_namespace(user_pk=None, count_nodes=False): diff --git a/tests/backends/aiida_django/migrations/test_migrations_many.py b/tests/backends/aiida_django/migrations/test_migrations_many.py index 06f3fe2d88..3039f59adb 100644 --- a/tests/backends/aiida_django/migrations/test_migrations_many.py +++ b/tests/backends/aiida_django/migrations/test_migrations_many.py @@ -7,7 +7,7 @@ # For further information on the license, see the LICENSE.txt file # # For further information please visit http://www.aiida.net # ########################################################################### -# pylint: disable=invalid-name, import-error, no-name-in-module +# pylint: disable=invalid-name """ This file contains the majority of the migration tests that are too short to go to a separate file. @@ -76,7 +76,7 @@ class TestDuplicateNodeUuidMigration(TestMigrations): def setUpBeforeMigration(self): from aiida.common.utils import get_new_uuid - from aiida.backends.general.migrations.utils import deduplicate_uuids, verify_uuid_uniqueness + from aiida.backends.general.migrations.duplicate_uuids import deduplicate_uuids, verify_uuid_uniqueness self.file_name = 'test.temp' self.file_content = '#!/bin/bash\n\necho test run\n' @@ -112,12 +112,12 @@ def setUpBeforeMigration(self): # Now run the function responsible for solving duplicate UUIDs which would also be called by the user # through the `verdi database integrity detect-duplicate-uuid` command - deduplicate_uuids(table='db_dbnode') + deduplicate_uuids(table='db_dbnode', dry_run=False) def test_deduplicated_uuids(self): """Verify that after the migration, all expected nodes are still there with unique UUIDs.""" # If the duplicate UUIDs were successfully fixed, the following should not raise. - from aiida.backends.general.migrations.utils import verify_uuid_uniqueness + from aiida.backends.general.migrations.duplicate_uuids import verify_uuid_uniqueness verify_uuid_uniqueness(table='db_dbnode') diff --git a/tests/orm/test_querybuilder.py b/tests/orm/test_querybuilder.py index 5ad1345a8b..78991bc07a 100644 --- a/tests/orm/test_querybuilder.py +++ b/tests/orm/test_querybuilder.py @@ -1130,10 +1130,22 @@ class QueryBuilderPath: def init_db(self, clear_database_before_test, backend): self.backend = backend + @staticmethod + def get_all_parents(node_pks, return_values=('id',)): + """Get all the parents of given nodes + + :param node_pks: one node pk or an iterable of node pks + :return: a list of aiida objects with all the parents of the nodes""" + from aiida.orm import Node, QueryBuilder + + q_build = QueryBuilder() + q_build.append(Node, tag='low_node', filters={'id': {'in': node_pks}}) + q_build.append(Node, with_descendants='low_node', project=return_values) + return q_build.all() + def test_query_path(self): # pylint: disable=too-many-statements - q = self.backend.query_manager n1 = orm.Data() n1.label = 'n1' n2 = orm.CalculationNode() @@ -1168,13 +1180,13 @@ def test_query_path(self): node.store() # There are no parents to n9, checking that - assert set([]) == set(q.get_all_parents([n9.pk])) + assert set([]) == set(self.get_all_parents([n9.pk])) # There is one parent to n6 - assert {(_,) for _ in (n6.pk,)} == {tuple(_) for _ in q.get_all_parents([n7.pk])} + assert {(_,) for _ in (n6.pk,)} == {tuple(_) for _ in self.get_all_parents([n7.pk])} # There are several parents to n4 - assert {(_.pk,) for _ in (n1, n2)} == {tuple(_) for _ in q.get_all_parents([n4.pk])} + assert {(_.pk,) for _ in (n1, n2)} == {tuple(_) for _ in self.get_all_parents([n4.pk])} # There are several parents to n5 - assert {(_.pk,) for _ in (n1, n2, n3, n4)} == {tuple(_) for _ in q.get_all_parents([n5.pk])} + assert {(_.pk,) for _ in (n1, n2, n3, n4)} == {tuple(_) for _ in self.get_all_parents([n5.pk])} # Yet, no links from 1 to 8 assert orm.QueryBuilder().append(orm.Node, filters={ @@ -1376,7 +1388,7 @@ def store_and_add(n, statistics): statistics['types'][n._plugin_type_string] += 1 # pylint: disable=no-member statistics['ctime_by_day'][n.ctime.strftime('%Y-%m-%d')] += 1 - qmanager = self.backend.query_manager + qmanager = self.backend.query() current_db_statistics = qmanager.get_creation_statistics() types = defaultdict(int) types.update(current_db_statistics['types']) @@ -1413,7 +1425,7 @@ def store_and_add(n, statistics): statistics['types'][n._plugin_type_string] += 1 # pylint: disable=no-member,protected-access statistics['ctime_by_day'][n.ctime.strftime('%Y-%m-%d')] += 1 - current_db_statistics = self.backend.query_manager.get_creation_statistics() + current_db_statistics = self.backend.query().get_creation_statistics() types = defaultdict(int) types.update(current_db_statistics['types']) ctime_by_day = defaultdict(int) @@ -1426,7 +1438,7 @@ def store_and_add(n, statistics): store_and_add(orm.Dict(), expected_db_statistics) store_and_add(orm.CalculationNode(), expected_db_statistics) - new_db_statistics = self.backend.query_manager.get_creation_statistics() + new_db_statistics = self.backend.query().get_creation_statistics() # I only check a few fields new_db_statistics = {k: v for k, v in new_db_statistics.items() if k in expected_db_statistics}