Config: warn when the configuration directory is created (#4896)

The configuration directory is automatically created when it is loaded
and it doesn't exist yet. This is useful for new installations, however,
it can lead to problems when users accidentally make a mistake when
setting an explicit path with the `AIIDA_PATH` environment variable. If
a type is present in the path, instead of the intended existing config
being used, a new empty one is created.

We add a warning to when the base configuration directory is created.
Note that this will also be emitted if the `AIIDA_PATH` variable is not
set, but even if the directory is created for the default setup this may
be useful information for the user.

The `aiida.manage.configuration.settings` module is also migrated to use
the `pathlib` module for file system path manipulations.

.github/system_tests/test_profile_manager.py

@@ -14,6 +14,7 @@
 import sys
 
 from pgtest import pgtest
+import pytest
 
 from aiida.manage.tests import TemporaryProfileManager, TestManagerError, get_test_backend_name
 from aiida.common.utils import Capturing
@@ -42,6 +43,7 @@ def test_create_aiida_db(self):
         self.profile_manager.create_aiida_db()
         self.assertTrue(self.profile_manager.postgres.db_exists(self.profile_manager.profile_info['database_name']))
 
+    @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder')
     def test_create_use_destroy_profile2(self):
         """
         Test temporary test profile creation

.github/system_tests/test_test_manager.py

@@ -12,6 +12,8 @@
 import warnings
 import sys
 
+import pytest
+
 from aiida.manage.tests import TestManager, get_test_backend_name
 
 
@@ -29,6 +31,7 @@ def setUp(self):
     def tearDown(self):
         self.test_manager.destroy_all()
 
+    @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder')
     def test_pgtest_argument(self):
         """
         Create a temporary profile, passing the pgtest argument.

aiida/manage/configuration/settings.py

@@ -8,9 +8,10 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 """Base settings required for the configuration of an AiiDA instance."""
-
-import errno
 import os
+import pathlib
+import typing
+import warnings
 
 USE_TZ = True
 
@@ -24,9 +25,9 @@
 DEFAULT_DAEMON_DIR_NAME = 'daemon'
 DEFAULT_DAEMON_LOG_DIR_NAME = 'log'
 
-AIIDA_CONFIG_FOLDER = None
-DAEMON_DIR = None
-DAEMON_LOG_DIR = None
+AIIDA_CONFIG_FOLDER: typing.Optional[pathlib.Path] = None
+DAEMON_DIR: typing.Optional[pathlib.Path] = None
+DAEMON_LOG_DIR: typing.Optional[pathlib.Path] = None
 
 
 def create_instance_directories():
@@ -35,32 +36,26 @@ def create_instance_directories():
     This will create the base AiiDA directory defined by the AIIDA_CONFIG_FOLDER variable, unless it already exists.
     Subsequently, it will create the daemon directory within it and the daemon log directory.
     """
-    directory_base = os.path.expanduser(AIIDA_CONFIG_FOLDER)
-    directory_daemon = os.path.join(directory_base, DAEMON_DIR)
-    directory_daemon_log = os.path.join(directory_base, DAEMON_LOG_DIR)
+    from aiida.common import ConfigurationError
+
+    directory_base = pathlib.Path(AIIDA_CONFIG_FOLDER).expanduser()
+    directory_daemon = directory_base / DAEMON_DIR
+    directory_daemon_log = directory_base / DAEMON_LOG_DIR
 
     umask = os.umask(DEFAULT_UMASK)
 
     try:
-        create_directory(directory_base)
-        create_directory(directory_daemon)
-        create_directory(directory_daemon_log)
-    finally:
-        os.umask(umask)
-
+        for path in [directory_base, directory_daemon, directory_daemon_log]:
 
-def create_directory(path):
-    """Attempt to create the configuration folder at the given path skipping if it already exists
+            if path is directory_base and not path.exists():
+                warnings.warn(f'Creating AiiDA configuration folder `{path}`.')
 
-    :param path: an absolute path to create a directory at
-    """
-    from aiida.common import ConfigurationError
-
-    try:
-        os.makedirs(path)
-    except OSError as exception:
-        if exception.errno != errno.EEXIST:
-            raise ConfigurationError(f"could not create the '{path}' configuration directory")
+            try:
+                path.mkdir(parents=True, exist_ok=True)
+            except OSError as exc:
+                raise ConfigurationError(f'could not create the `{path}` configuration directory: {exc}') from exc
+    finally:
+        os.umask(umask)
 
 
 def set_configuration_directory():
@@ -86,25 +81,25 @@ def set_configuration_directory():
     if environment_variable:
 
         # Loop over all the paths in the `AIIDA_PATH` variable to see if any of them contain a configuration folder
-        for base_dir_path in [os.path.expanduser(path) for path in environment_variable.split(':') if path]:
+        for base_dir_path in [path for path in environment_variable.split(':') if path]:
 
-            AIIDA_CONFIG_FOLDER = os.path.expanduser(os.path.join(base_dir_path))
+            AIIDA_CONFIG_FOLDER = pathlib.Path(base_dir_path).expanduser()
 
             # Only add the base config directory name to the base path if it does not already do so
             # Someone might already include it in the environment variable. e.g.: AIIDA_PATH=/home/some/path/.aiida
-            if not AIIDA_CONFIG_FOLDER.endswith(DEFAULT_CONFIG_DIR_NAME):
-                AIIDA_CONFIG_FOLDER = os.path.join(AIIDA_CONFIG_FOLDER, DEFAULT_CONFIG_DIR_NAME)
+            if AIIDA_CONFIG_FOLDER.name != DEFAULT_CONFIG_DIR_NAME:
+                AIIDA_CONFIG_FOLDER = AIIDA_CONFIG_FOLDER / DEFAULT_CONFIG_DIR_NAME
 
             # If the directory exists, we leave it set and break the loop
-            if os.path.isdir(AIIDA_CONFIG_FOLDER):
+            if AIIDA_CONFIG_FOLDER.is_dir():
                 break
 
     else:
         # The `AIIDA_PATH` variable is not set, so default to the default path and try to create it if it does not exist
-        AIIDA_CONFIG_FOLDER = os.path.expanduser(os.path.join(DEFAULT_AIIDA_PATH, DEFAULT_CONFIG_DIR_NAME))
+        AIIDA_CONFIG_FOLDER = pathlib.Path(DEFAULT_AIIDA_PATH).expanduser() / DEFAULT_CONFIG_DIR_NAME
 
-    DAEMON_DIR = os.path.join(AIIDA_CONFIG_FOLDER, DEFAULT_DAEMON_DIR_NAME)
-    DAEMON_LOG_DIR = os.path.join(DAEMON_DIR, DEFAULT_DAEMON_LOG_DIR_NAME)
+    DAEMON_DIR = AIIDA_CONFIG_FOLDER / DEFAULT_DAEMON_DIR_NAME
+    DAEMON_LOG_DIR = DAEMON_DIR / DEFAULT_DAEMON_LOG_DIR_NAME
 
     create_instance_directories()
 

docs/source/howto/installation.rst

@@ -435,8 +435,8 @@ Backing up your installation
 
 A full backup of an AiiDA instance and AiiDA managed data requires a backup of:
 
-* the profile configuration in the ``config.json`` file located in the ``.aiida`` folder.
-  Typically located at ``~/.aiida`` (see also :ref:`intro:install:setup`).
+* the AiiDA configuration folder, which is typically named ``.aiida`` and located in the home folder (see also :ref:`intro:install:setup`).
+  This folder contains, among other things, the ``config.json`` configuration file and log files.
 
 * files associated with nodes in the repository folder (one per profile). Typically located in the ``.aiida`` folder.
 

tests/manage/configuration/test_config.py

@@ -9,9 +9,12 @@
 ###########################################################################
 """Tests for the Config class."""
 import os
+import pathlib
 import shutil
 import tempfile
 
+import pytest
+
 from aiida.backends.testbase import AiidaTestCase
 from aiida.common import exceptions, json
 from aiida.manage.configuration import Config, Profile, settings
@@ -42,6 +45,7 @@ def tearDownClass(cls, *args, **kwargs):
             except KeyError:
                 pass
 
+    @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder')
     def test_environment_variable_not_set(self):
         """Check that if the environment variable is not set, config folder will be created in `DEFAULT_AIIDA_PATH`.
 
@@ -63,10 +67,11 @@ def test_environment_variable_not_set(self):
 
             config_folder = os.path.join(directory, settings.DEFAULT_CONFIG_DIR_NAME)
             self.assertTrue(os.path.isdir(config_folder))
-            self.assertEqual(settings.AIIDA_CONFIG_FOLDER, config_folder)
+            self.assertEqual(settings.AIIDA_CONFIG_FOLDER, pathlib.Path(config_folder))
         finally:
             shutil.rmtree(directory)
 
+    @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder')
     def test_environment_variable_set_single_path_without_config_folder(self):  # pylint: disable=invalid-name
         """If `AIIDA_PATH` is set but does not contain a configuration folder, it should be created."""
         try:
@@ -80,11 +85,12 @@ def test_environment_variable_set_single_path_without_config_folder(self):  # py
             # This should have created the configuration directory in the path
             config_folder = os.path.join(directory, settings.DEFAULT_CONFIG_DIR_NAME)
             self.assertTrue(os.path.isdir(config_folder))
-            self.assertEqual(settings.AIIDA_CONFIG_FOLDER, config_folder)
+            self.assertEqual(settings.AIIDA_CONFIG_FOLDER, pathlib.Path(config_folder))
 
         finally:
             shutil.rmtree(directory)
 
+    @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder')
     def test_environment_variable_set_single_path_with_config_folder(self):  # pylint: disable=invalid-name
         """If `AIIDA_PATH` is set and already contains a configuration folder it should simply be used."""
         try:
@@ -99,10 +105,11 @@ def test_environment_variable_set_single_path_with_config_folder(self):  # pylin
             # This should have created the configuration directory in the pathpath
             config_folder = os.path.join(directory, settings.DEFAULT_CONFIG_DIR_NAME)
             self.assertTrue(os.path.isdir(config_folder))
-            self.assertEqual(settings.AIIDA_CONFIG_FOLDER, config_folder)
+            self.assertEqual(settings.AIIDA_CONFIG_FOLDER, pathlib.Path(config_folder))
         finally:
             shutil.rmtree(directory)
 
+    @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder')
     def test_environment_variable_path_including_config_folder(self):  # pylint: disable=invalid-name
         """If `AIIDA_PATH` is set and the path contains the base name of the config folder, it should work, i.e:
 
@@ -122,11 +129,12 @@ def test_environment_variable_path_including_config_folder(self):  # pylint: dis
             # This should have created the configuration directory in the pathpath
             config_folder = os.path.join(directory, settings.DEFAULT_CONFIG_DIR_NAME)
             self.assertTrue(os.path.isdir(config_folder))
-            self.assertEqual(settings.AIIDA_CONFIG_FOLDER, config_folder)
+            self.assertEqual(settings.AIIDA_CONFIG_FOLDER, pathlib.Path(config_folder))
 
         finally:
             shutil.rmtree(directory)
 
+    @pytest.mark.filterwarnings('ignore:Creating AiiDA configuration folder')
     def test_environment_variable_set_multiple_path(self):  # pylint: disable=invalid-name
         """If `AIIDA_PATH` is set with multiple paths without actual config folder, one is created in the last."""
         try:
@@ -142,7 +150,7 @@ def test_environment_variable_set_multiple_path(self):  # pylint: disable=invali
             # This should have created the configuration directory in the last path
             config_folder = os.path.join(directory_c, settings.DEFAULT_CONFIG_DIR_NAME)
             self.assertTrue(os.path.isdir(config_folder))
-            self.assertEqual(settings.AIIDA_CONFIG_FOLDER, config_folder)
+            self.assertEqual(settings.AIIDA_CONFIG_FOLDER, pathlib.Path(config_folder))
 
         finally:
             shutil.rmtree(directory_a)