Improve lockfiles

.github/workflows/main.yml

@@ -45,7 +45,7 @@ jobs:
         run: |
           conda config --add channels conda-forge
           conda config --set channel_priority strict
-          conda install -n base python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cxx-compiler ccache cmake gtest gmock reproc-cpp yaml-cpp
+          conda install -n base python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json "cpp-filesystem>=1.5.8" conda cxx-compiler ccache cmake gtest gmock reproc-cpp yaml-cpp
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -147,7 +147,7 @@ jobs:
           export MAMBA_ROOT_PREFIX=~/mambaroot
           export MAMBA_EXE=$(pwd)/micromamba
           . $MAMBA_ROOT_PREFIX/etc/profile.d/mamba.sh
-          micromamba create -y -p ~/build_env pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cxx-compiler ccache cmake gtest gmock cpp-filesystem reproc-cpp yaml-cpp cli11 -c conda-forge
+          micromamba create -y -p ~/build_env pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cxx-compiler ccache cmake gtest gmock "cpp-filesystem>=1.5.8" reproc-cpp yaml-cpp cli11 -c conda-forge
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
       - name: build tests
@@ -169,6 +169,7 @@ jobs:
                    -DBUILD_BINDINGS=OFF \
                    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                    -DCMAKE_C_COMPILER_LAUNCHER=ccache
+          make test_mamba_lock -j2
           make test -j2
 
   umamba_tests:
@@ -208,7 +209,7 @@ jobs:
           export MAMBA_ROOT_PREFIX=~/mambaroot
           export MAMBA_EXE=$(pwd)/micromamba
           . $MAMBA_ROOT_PREFIX/etc/profile.d/mamba.sh
-          micromamba create -y -p ~/build_env pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cxx-compiler ccache cmake gtest gmock cpp-filesystem reproc-cpp yaml-cpp pyyaml cli11 -c conda-forge
+          micromamba create -y -p ~/build_env pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cxx-compiler ccache cmake gtest gmock "cpp-filesystem>=1.5.8" reproc-cpp yaml-cpp pyyaml cli11 -c conda-forge
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
       - name: build micromamba
@@ -323,7 +324,7 @@ jobs:
         run: |
           conda config --add channels conda-forge
           conda config --set channel_priority strict
-          conda install -n base -q -y vs2017_win-64 ccache python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest gmock ninja reproc-cpp yaml-cpp cli11
+          conda install -n base -q -y vs2017_win-64 ccache python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json "cpp-filesystem>=1.5.8" conda cmake gtest gmock ninja reproc-cpp yaml-cpp cli11
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -398,7 +399,7 @@ jobs:
         run: |
           conda config --add channels conda-forge
           conda config --set channel_priority strict
-          conda create -q -y -n mamba-tests vs2017_win-64 ccache python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest gmock ninja reproc-cpp yaml-cpp cli11 pytest
+          conda create -q -y -n mamba-tests vs2017_win-64 ccache python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json "cpp-filesystem>=1.5.8" conda cmake gtest gmock ninja reproc-cpp yaml-cpp cli11 pytest
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
       - name: Run C++ tests Windows
@@ -412,8 +413,8 @@ jobs:
                    -G "Ninja" ^
                    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ^
                    -DCMAKE_C_COMPILER_LAUNCHER=ccache
-
-          ninja test
+          ninja test_mamba_lock -j2
+          ninja test -j2
 
   umamba_tests_win:
     runs-on: ${{ matrix.os }}
@@ -508,7 +509,7 @@ jobs:
         run: |
           conda config --add channels conda-forge
           conda config --set channel_priority strict
-          conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest gmock ninja reproc-cpp yaml-cpp pyyaml cli11 pytest pytest-lazy-fixture menuinst
+          conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json "cpp-filesystem>=1.5.8" conda cmake gtest gmock ninja reproc-cpp yaml-cpp pyyaml cli11 pytest pytest-lazy-fixture menuinst
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
       - name: micromamba python based tests
@@ -549,7 +550,7 @@ jobs:
         run: |
           conda config --add channels conda-forge
           conda config --set channel_priority strict
-          conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest gmock ninja reproc-cpp yaml-cpp pyyaml cli11 pytest pytest-lazy-fixture menuinst
+          conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json "cpp-filesystem>=1.5.8" conda cmake gtest gmock ninja reproc-cpp yaml-cpp pyyaml cli11 pytest pytest-lazy-fixture menuinst
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
       - name: micromamba python based tests with pwsh
@@ -588,7 +589,7 @@ jobs:
         run: |
           conda config --add channels conda-forge
           conda config --set channel_priority strict
-          conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest gmock ninja reproc-cpp yaml-cpp pyyaml cli11 pytest pytest-lazy-fixture menuinst
+          conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json "cpp-filesystem>=1.5.8" conda cmake gtest gmock ninja reproc-cpp yaml-cpp pyyaml cli11 pytest pytest-lazy-fixture menuinst
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
       - name: micromamba python based tests

environment-dev.yml

@@ -15,7 +15,7 @@ dependencies:
   - cxx-compiler
   - gtest
   - gmock
-  - cpp-filesystem
+  - cpp-filesystem >=1.5.8
   - reproc-cpp
   - yaml-cpp
   - pyyaml

include/mamba/api/clean.hpp

@@ -14,6 +14,7 @@ namespace mamba
     const int MAMBA_CLEAN_INDEX = 1 << 1;
     const int MAMBA_CLEAN_PKGS = 1 << 2;
     const int MAMBA_CLEAN_TARBALLS = 1 << 3;
+    const int MAMBA_CLEAN_LOCKS = 1 << 4;
 
     void clean(int options);
 }

include/mamba/core/context.hpp

@@ -163,6 +163,8 @@ namespace mamba
         bool no_rc = false;
         bool no_env = false;
 
+        std::size_t lock_timeout = 0;
+
         // Conda compat
         bool add_pip_as_python_dependency = true;
 

include/mamba/core/util.hpp

@@ -7,6 +7,11 @@
 #ifndef MAMBA_CORE_UTIL_HPP
 #define MAMBA_CORE_UTIL_HPP
 
+#include "mamba/core/mamba_fs.hpp"
+#include "mamba/core/output.hpp"
+
+#include "nlohmann/json.hpp"
+
 #include <array>
 #include <iomanip>
 #include <limits>
@@ -18,11 +23,7 @@
 #include <time.h>
 #include <vector>
 #include <regex>
-
-#include "nlohmann/json.hpp"
-
-#include "mamba_fs.hpp"
-#include "output.hpp"
+#include <chrono>
 
 
 namespace mamba
@@ -132,28 +133,72 @@ namespace mamba
         fs::path m_path;
     };
 
+    class Lock;
+
     class LockFile
     {
     public:
-        LockFile(const fs::path& path);
+        // LockFile(const fs::path& path);
+        LockFile(const fs::path& path, const std::chrono::seconds& timeout);
         ~LockFile();
 
-        LockFile(const LockFile&) = delete;
-        LockFile& operator=(const LockFile&) = delete;
-        LockFile& operator=(LockFile&&) = default;
+        int fd() const;
 
-        fs::path& path();
-        operator fs::path();
+#ifdef _WIN32
+        // Using file descriptor on Windows may cause false negative
+        static bool is_locked(const fs::path& path);
+#else
+        // Opening a new file descriptor on Unix would clear locks
+        static bool is_locked(int fd);
+#endif
+        static int read_pid(int fd);
+
+    private:
+        fs::path m_path;
+        std::chrono::seconds m_timeout;
+        int m_fd = -1;
+
+        bool set_lock(bool blocking) const;
+
+        int read_pid() const;
+        bool write_pid(int pid) const;
+        bool locked() const;
+        bool lock(int pid, bool blocking) const;
+        bool unlock() const;
+        int close_fd();
+        void remove() noexcept;
+
+        friend class Lock;
+    };
+
+    class Lock
+    {
+    public:
+        Lock(const fs::path& path);
+        ~Lock();
+
+        Lock(const Lock&) = delete;
+        Lock& operator=(const Lock&) = delete;
+        Lock& operator=(Lock&&) = default;
+
+        bool locked() const;
+        int fd() const;
+        fs::path path() const;
 
     private:
         fs::path m_path;
-        int m_fd;
+        fs::path m_lock;
+        std::unique_ptr<LockFile> p_lock_file;
+        bool m_locked;
 
 #if defined(__APPLE__) or defined(__linux__)
         pid_t m_pid;
 #else
         int m_pid;
 #endif
+
+        bool try_lock();
+        bool lock();
     };
 
     /*************************

src/api/clean.cpp

@@ -8,6 +8,7 @@
 #include "mamba/api/configuration.hpp"
 
 #include "mamba/core/context.hpp"
+#include "mamba/core/mamba_fs.hpp"
 #include "mamba/core/package_cache.hpp"
 
 
@@ -25,21 +26,22 @@ namespace mamba
         bool clean_index = options & MAMBA_CLEAN_INDEX;
         bool clean_pkgs = options & MAMBA_CLEAN_PKGS;
         bool clean_tarballs = options & MAMBA_CLEAN_TARBALLS;
+        bool clean_locks = options & MAMBA_CLEAN_LOCKS;
 
-        if (!(clean_all || clean_index || clean_pkgs || clean_tarballs))
+        if (!(clean_all || clean_index || clean_pkgs || clean_tarballs || clean_locks))
         {
-            std::cout << "Nothing to do." << std::endl;
+            Console::stream() << "Nothing to do." << std::endl;
             return;
         }
 
-        Console::print("Collect information..");
+        Console::stream() << "Collect information..";
 
         std::vector<fs::path> envs;
 
         MultiPackageCache caches(ctx.pkgs_dirs);
         if (!ctx.dry_run && (clean_index || clean_all))
         {
-            Console::print("Cleaning index cache..");
+            Console::stream() << "Cleaning index cache..";
 
             for (auto* pkg_cache : caches.writable_caches())
                 if (fs::exists(pkg_cache->get_pkgs_dir() / "cache"))
@@ -55,6 +57,50 @@ namespace mamba
                 }
         }
 
+        if (!ctx.dry_run && (clean_locks || clean_all))
+        {
+            Console::stream() << "Cleaning lock files..";
+
+            for (auto* pkg_cache : caches.writable_caches())
+            {
+                if (fs::exists(pkg_cache->get_pkgs_dir()))
+                    for (auto& p : fs::directory_iterator(pkg_cache->get_pkgs_dir()))
+                    {
+                        if (p.exists() && ends_with(p.path().string(), ".lock")
+                            && fs::exists(rstrip(p.path().string(), ".lock")))
+                        {
+                            try
+                            {
+                                fs::remove(p);
+                            }
+                            catch (...)
+                            {
+                                LOG_WARNING << "Could not clean lock file '" << p.path().string()
+                                            << "'";
+                            }
+                        }
+                    }
+
+                if (fs::exists(pkg_cache->get_pkgs_dir() / "cache"))
+                    for (auto& p :
+                         fs::recursive_directory_iterator(pkg_cache->get_pkgs_dir() / "cache"))
+                    {
+                        if (p.exists() && ends_with(p.path().string(), ".lock"))
+                        {
+                            try
+                            {
+                                fs::remove(p);
+                            }
+                            catch (...)
+                            {
+                                LOG_WARNING << "Could not clean lock file '" << p.path().string()
+                                            << "'";
+                            }
+                        }
+                    }
+            }
+        }
+
         if (fs::exists(ctx.root_prefix / "conda-meta"))
         {
             envs.push_back(ctx.root_prefix);

src/api/configuration.cpp

@@ -814,6 +814,16 @@ namespace mamba
                         Spend extra time validating package contents. It consists of running
                         cryptographic verifications on channels and packages metadata.)")));
 
+        insert(Configurable("lock_timeout", &ctx.lock_timeout)
+                   .group("Link & Install")
+                   .set_rc_configurable()
+                   .set_env_var_names()
+                   .description("Lock timeout")
+                   .long_description(unindent(R"(
+                        Lock timeout for blocking mode when waiting for another process
+                        to release the path.
+                        Not configurable on Windows: set to 10s by WinAPI)")));
+
         // Output, Prompt and Flow
         insert(Configurable("always_yes", &ctx.always_yes)
                    .group("Output, Prompt and Flow Control")

src/api/install.cpp

@@ -339,10 +339,10 @@ namespace mamba
 
         std::vector<std::shared_ptr<MSubdirData>> subdirs;
         MultiDownloadTarget multi_dl;
-        std::unique_ptr<LockFile> subdir_download_lock;
+        std::unique_ptr<Lock> subdir_download_lock;
         if (!ctx.offline)
         {
-            subdir_download_lock = std::make_unique<LockFile>(cache_dir / "mamba.lock");
+            subdir_download_lock = std::make_unique<Lock>(cache_dir);
         }
 
         std::vector<std::pair<int, int>> priorities;
@@ -535,7 +535,7 @@ namespace mamba
                 detail::create_target_directory(ctx.target_prefix);
             }
             {
-                LockFile(pkgs_dirs / "mamba.lock");
+                Lock pkgs_dirs_lock(pkgs_dirs);
                 trans.execute(prefix_data);
             }
             for (auto other_spec : config.at("others_pkg_mgrs_specs")
@@ -774,7 +774,7 @@ namespace mamba
                 fs::create_directories(pkgs_dirs);
             }
 
-            LockFile(pkgs_dirs / "mamba.lock");
+            Lock pkgs_dirs_lock(pkgs_dirs);
 
             std::vector<std::unique_ptr<PackageDownloadExtractTarget>> targets;
             MultiDownloadTarget multi_dl;