From 4f635db86af2830c8b3b1f0be00cccecc011f32e Mon Sep 17 00:00:00 2001 From: Mathieu Pellerin Date: Sat, 31 Aug 2024 17:50:22 +0700 Subject: [PATCH] [network] Implement a smart cache size --- src/app/options/qgsoptions.cpp | 16 ++-- src/core/network/qgsnetworkaccessmanager.cpp | 6 +- src/core/network/qgsnetworkdiskcache.cpp | 83 +++++++++++++++++++ src/core/network/qgsnetworkdiskcache.h | 10 ++- src/core/network/qgsrangerequestcache.cpp | 7 ++ src/core/qgstiledownloadmanager.cpp | 7 +- src/core/settings/qgssettingsregistrycore.cpp | 4 + src/core/settings/qgssettingsregistrycore.h | 7 ++ src/core/settings/qgssettingstree.h | 1 + src/server/qgsserver.cpp | 3 +- src/server/qgsserversettings.cpp | 8 +- src/ui/qgsoptionsbase.ui | 29 ++++++- tests/src/python/test_qgsserver_settings.py | 2 +- tests/testdata/qgis_server_settings/conf0.ini | 2 +- tests/testdata/qgis_server_settings/conf1.ini | 2 +- 15 files changed, 163 insertions(+), 24 deletions(-) diff --git a/src/app/options/qgsoptions.cpp b/src/app/options/qgsoptions.cpp index 84084ae7e174..385ff989aa88 100644 --- a/src/app/options/qgsoptions.cpp +++ b/src/app/options/qgsoptions.cpp @@ -488,14 +488,14 @@ QgsOptions::QgsOptions( QWidget *parent, Qt::WindowFlags fl, const QListsetText( mSettings->value( QStringLiteral( "cache/directory" ) ).toString() ); + mCacheDirectory->setText( QgsSettingsRegistryCore::settingsNetworkCacheDirectory->value() ); mCacheDirectory->setPlaceholderText( QStandardPaths::writableLocation( QStandardPaths::CacheLocation ) ); mCacheSize->setMinimum( 0 ); mCacheSize->setMaximum( std::numeric_limits::max() ); - mCacheSize->setSingleStep( 1024 ); - qint64 cacheSize = mSettings->value( QStringLiteral( "cache/size" ), 256 * 1024 * 1024 ).toLongLong(); - mCacheSize->setValue( static_cast( cacheSize / 1024 ) ); - mCacheSize->setClearValue( 50 * 1024 ); + mCacheSize->setSingleStep( 50 ); + qint64 cacheSize = QgsSettingsRegistryCore::settingsNetworkCacheSize->value(); + mCacheSize->setValue( static_cast( cacheSize / 1024 / 1024 ) ); + mCacheSize->setClearValue( 0 ); connect( mBrowseCacheDirectory, &QAbstractButton::clicked, this, &QgsOptions::browseCacheDirectory ); connect( mClearCache, &QAbstractButton::clicked, this, &QgsOptions::clearCache ); @@ -1584,11 +1584,11 @@ void QgsOptions::saveOptions() mSettings->setValue( QStringLiteral( "proxy/proxyType" ), mProxyTypeComboBox->currentText() ); if ( !mCacheDirectory->text().isEmpty() ) - mSettings->setValue( QStringLiteral( "cache/directory" ), mCacheDirectory->text() ); + QgsSettingsRegistryCore::settingsNetworkCacheDirectory->setValue( mCacheDirectory->text() ); else - mSettings->remove( QStringLiteral( "cache/directory" ) ); + QgsSettingsRegistryCore::settingsNetworkCacheDirectory->remove(); - mSettings->setValue( QStringLiteral( "cache/size" ), QVariant::fromValue( mCacheSize->value() * 1024LL ) ); + QgsSettingsRegistryCore::settingsNetworkCacheSize->setValue( mCacheSize->value() * 1024LL * 1024LL ); //url with no proxy at all QStringList noProxyUrls; diff --git a/src/core/network/qgsnetworkaccessmanager.cpp b/src/core/network/qgsnetworkaccessmanager.cpp index cf297b32101e..6bc6869e9298 100644 --- a/src/core/network/qgsnetworkaccessmanager.cpp +++ b/src/core/network/qgsnetworkaccessmanager.cpp @@ -24,6 +24,7 @@ #include "qgsapplication.h" #include "qgsmessagelog.h" #include "qgssettings.h" +#include "qgssettingsregistrycore.h" #include "qgslogger.h" #include "qgis.h" #include "qgsnetworkdiskcache.h" @@ -749,12 +750,13 @@ void QgsNetworkAccessManager::setupDefaultProxyAndCache( Qt::ConnectionType conn if ( !newcache ) newcache = new QgsNetworkDiskCache( this ); - QString cacheDirectory = settings.value( QStringLiteral( "cache/directory" ) ).toString(); + QString cacheDirectory = QgsSettingsRegistryCore::settingsNetworkCacheDirectory->value(); if ( cacheDirectory.isEmpty() ) cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation ); - const qint64 cacheSize = settings.value( QStringLiteral( "cache/size" ), 256 * 1024 * 1024 ).toLongLong(); newcache->setCacheDirectory( cacheDirectory ); + qint64 cacheSize = QgsSettingsRegistryCore::settingsNetworkCacheSize->value(); newcache->setMaximumCacheSize( cacheSize ); + QgsDebugMsgLevel( QStringLiteral( "cacheDirectory: %1" ).arg( newcache->cacheDirectory() ), 4 ); QgsDebugMsgLevel( QStringLiteral( "maximumCacheSize: %1" ).arg( newcache->maximumCacheSize() ), 4 ); diff --git a/src/core/network/qgsnetworkdiskcache.cpp b/src/core/network/qgsnetworkdiskcache.cpp index 3e4c78cf8f2b..e391f98805d1 100644 --- a/src/core/network/qgsnetworkdiskcache.cpp +++ b/src/core/network/qgsnetworkdiskcache.cpp @@ -18,6 +18,9 @@ #include "qgsnetworkdiskcache.h" +#include +#include + ///@cond PRIVATE ExpirableNetworkDiskCache QgsNetworkDiskCache::sDiskCache; ///@endcond @@ -49,6 +52,13 @@ qint64 QgsNetworkDiskCache::maximumCacheSize() const void QgsNetworkDiskCache::setMaximumCacheSize( qint64 size ) { const QMutexLocker lock( &sDiskCacheMutex ); + + if ( size == 0 ) + { + // Calculate maximum cache size based on available free space + size = smartCacheSize( sDiskCache.cacheDirectory() ); + } + sDiskCache.setMaximumCacheSize( size ); } @@ -111,3 +121,76 @@ void QgsNetworkDiskCache::clear() const QMutexLocker lock( &sDiskCacheMutex ); return sDiskCache.clear(); } + +qint64 QgsNetworkDiskCache::smartCacheSize( const QString &cacheDir ) +{ + static qint64 cacheSize = 0; + static std::once_flag initialized; + std::call_once( initialized, [ = ] + { + std::function dirSize; + dirSize = [&dirSize]( const QString & dirPath ) -> qint64 + { + qint64 size = 0; + QDir dir( dirPath ); + + const QStringList filePaths = dir.entryList( QDir::Files | QDir::System | QDir::Hidden ); + for ( const QString &filePath : filePaths ) + { + QFileInfo fi( dir, filePath ); + size += fi.size(); + } + + const QStringList childDirPaths = dir.entryList( QDir::Dirs | QDir::NoDotAndDotDot | QDir::System | QDir::Hidden | QDir::NoSymLinks ); + for ( const QString &childDirPath : childDirPaths ) + { + size += dirSize( dirPath + QDir::separator() + childDirPath ); + } + + return size; + }; + + qint64 bytesFree; + QStorageInfo storageInfo( cacheDir ); + bytesFree = storageInfo.bytesFree() + dirSize( cacheDir ); + + // NOLINTBEGIN(bugprone-narrowing-conversions) + // Logic taken from Firefox's smart cache size handling + qint64 available10MB = bytesFree / 1024 / ( 1024LL * 10 ); + qint64 cacheSize10MB = 0; + if ( available10MB > 2500 ) + { + // Cap the cache size to 1GB + cacheSize10MB = 100; + } + else + { + if ( available10MB > 700 ) + { + // Add 2.5% of the free space above 7GB + cacheSize10MB += ( available10MB - 700 ) * 0.025; + available10MB = 700; + } + if ( available10MB > 50 ) + { + // Add 7.5% of free space between 500MB to 7GB + cacheSize10MB += ( available10MB - 50 ) * 0.075; + available10MB = 50; + } + +#if defined( Q_OS_ANDROID ) + // On Android, smaller/older devices may have very little storage + + // Add 16% of free space up to 500 MB + cacheSize10MB += std::max( 2LL, static_cast( available10MB * 0.16 ) ); +#else + // Add 30% of free space up to 500 MB + cacheSize10MB += std::max( 5LL, static_cast( available10MB * 0.30 ) ); +#endif + } + cacheSize = cacheSize10MB * 10 * 1024 * 1024; + // NOLINTEND(bugprone-narrowing-conversions) + } ); + + return cacheSize; +} diff --git a/src/core/network/qgsnetworkdiskcache.h b/src/core/network/qgsnetworkdiskcache.h index b666f0bdac20..776d4ddc6dae 100644 --- a/src/core/network/qgsnetworkdiskcache.h +++ b/src/core/network/qgsnetworkdiskcache.h @@ -20,6 +20,8 @@ #define SIP_NO_FILE +#include "qgis_core.h" + #include #include @@ -45,7 +47,7 @@ class ExpirableNetworkDiskCache : public QNetworkDiskCache * * \note not available in Python bindings */ -class QgsNetworkDiskCache : public QNetworkDiskCache +class CORE_EXPORT QgsNetworkDiskCache : public QNetworkDiskCache { Q_OBJECT @@ -87,6 +89,12 @@ class QgsNetworkDiskCache : public QNetworkDiskCache //! \see QNetworkDiskCache::fileMetaData() QNetworkCacheMetaData fileMetaData( const QString &fileName ) const; + /** + * Returns a smart cache size, in bytes, based on available free space + * \since QGIS 3.40 + */ + static qint64 smartCacheSize( const QString &path ); + public slots: //! \see QNetworkDiskCache::clear() void clear() override; diff --git a/src/core/network/qgsrangerequestcache.cpp b/src/core/network/qgsrangerequestcache.cpp index b3681b80e43c..216cef14b67d 100644 --- a/src/core/network/qgsrangerequestcache.cpp +++ b/src/core/network/qgsrangerequestcache.cpp @@ -16,6 +16,7 @@ ***************************************************************************/ #include "qgsrangerequestcache.h" +#include "qgsnetworkdiskcache.h" #include #include @@ -75,6 +76,12 @@ bool QgsRangeRequestCache::setCacheDirectory( const QString &path ) void QgsRangeRequestCache::setCacheSize( qint64 cacheSize ) { + if ( cacheSize == 0 ) + { + // Calculate maximum cache size based on available free space + cacheSize = QgsNetworkDiskCache::smartCacheSize( mCacheDir ); + } + mMaxDataSize = cacheSize; expire(); } diff --git a/src/core/qgstiledownloadmanager.cpp b/src/core/qgstiledownloadmanager.cpp index a0a906cdacdc..bd2ce162295e 100644 --- a/src/core/qgstiledownloadmanager.cpp +++ b/src/core/qgstiledownloadmanager.cpp @@ -21,6 +21,8 @@ #include "qgsnetworkaccessmanager.h" #include "qgsrangerequestcache.h" #include "qgssettings.h" +#include "qgssettingsregistrycore.h" +#include "qgssettingsentryimpl.h" #include #include @@ -190,7 +192,7 @@ QgsTileDownloadManager::QgsTileDownloadManager() mRangesCache.reset( new QgsRangeRequestCache ); const QgsSettings settings; - QString cacheDirectory = settings.value( QStringLiteral( "cache/directory" ) ).toString(); + QString cacheDirectory = QgsSettingsRegistryCore::settingsNetworkCacheDirectory->value(); if ( cacheDirectory.isEmpty() ) cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation ); if ( !cacheDirectory.endsWith( QDir::separator() ) ) @@ -198,9 +200,8 @@ QgsTileDownloadManager::QgsTileDownloadManager() cacheDirectory.push_back( QDir::separator() ); } cacheDirectory += QLatin1String( "http-ranges" ); - const qint64 cacheSize = settings.value( QStringLiteral( "cache/size" ), 256 * 1024 * 1024 ).toLongLong(); - mRangesCache->setCacheDirectory( cacheDirectory ); + qint64 cacheSize = QgsSettingsRegistryCore::settingsNetworkCacheSize->value(); mRangesCache->setCacheSize( cacheSize ); } diff --git a/src/core/settings/qgssettingsregistrycore.cpp b/src/core/settings/qgssettingsregistrycore.cpp index 024d4d0ab4d4..d317f7d858bc 100644 --- a/src/core/settings/qgssettingsregistrycore.cpp +++ b/src/core/settings/qgssettingsregistrycore.cpp @@ -116,6 +116,10 @@ const QgsSettingsEntryInteger *QgsSettingsRegistryCore::settingsLayerParallelLoa const QgsSettingsEntryBool *QgsSettingsRegistryCore::settingsLayerParallelLoading = new QgsSettingsEntryBool( QStringLiteral( "provider-parallel-loading" ), QgsSettingsTree::sTreeCore, true, QStringLiteral( "Load layers in parallel (only available for some providers (GDAL and PostgreSQL)" ), Qgis::SettingsOption() ); +const QgsSettingsEntryString *QgsSettingsRegistryCore::settingsNetworkCacheDirectory = new QgsSettingsEntryString( QStringLiteral( "directory" ), QgsSettingsTree::sTreeNetworkCache, QString(), QStringLiteral( "Network disk cache directory" ) ); + +const QgsSettingsEntryInteger64 *QgsSettingsRegistryCore::settingsNetworkCacheSize = new QgsSettingsEntryInteger64( QStringLiteral( "size-bytes" ), QgsSettingsTree::sTreeNetworkCache, 0, QStringLiteral( "Network disk cache size in bytes (0 = automatic size)" ) ); + QgsSettingsRegistryCore::QgsSettingsRegistryCore() : QgsSettingsRegistry() { diff --git a/src/core/settings/qgssettingsregistrycore.h b/src/core/settings/qgssettingsregistrycore.h index dcc798368ea5..20dc1c7e64b3 100644 --- a/src/core/settings/qgssettingsregistrycore.h +++ b/src/core/settings/qgssettingsregistrycore.h @@ -26,6 +26,7 @@ class QgsSettingsEntryBool; class QgsSettingsEntryColor; class QgsSettingsEntryDouble; class QgsSettingsEntryInteger; +class QgsSettingsEntryInteger64; class QgsSettingsEntryString; class QgsSettingsEntryStringList; template class QgsSettingsEntryEnumFlag; @@ -175,6 +176,12 @@ class CORE_EXPORT QgsSettingsRegistryCore : public QgsSettingsRegistry //! Settings entry whether layer are loading in parallel static const QgsSettingsEntryBool *settingsLayerParallelLoading; + //! Settings entry network cache directory + static const QgsSettingsEntryString *settingsNetworkCacheDirectory; + + //! Settings entry network cache directory + static const QgsSettingsEntryInteger64 *settingsNetworkCacheSize; + private: friend class QgsApplication; diff --git a/src/core/settings/qgssettingstree.h b/src/core/settings/qgssettingstree.h index 1dc8ce95471f..36a6221d8c73 100644 --- a/src/core/settings/qgssettingstree.h +++ b/src/core/settings/qgssettingstree.h @@ -63,6 +63,7 @@ class CORE_EXPORT QgsSettingsTree static inline QgsSettingsTreeNode *sTreeWms = treeRoot()->createChildNode( QStringLiteral( "wms" ) ); static inline QgsSettingsTreeNode *sTreeMeasure = treeRoot()->createChildNode( QStringLiteral( "measure" ) ); static inline QgsSettingsTreeNode *sTreeAnnotations = treeRoot()->createChildNode( QStringLiteral( "annotations" ) ); + static inline QgsSettingsTreeNode *sTreeNetworkCache = treeRoot()->createChildNode( QStringLiteral( "cache" ) ); #endif diff --git a/src/server/qgsserver.cpp b/src/server/qgsserver.cpp index 245680757ba1..ee2f2b2683b6 100644 --- a/src/server/qgsserver.cpp +++ b/src/server/qgsserver.cpp @@ -31,6 +31,7 @@ #include "qgslogger.h" #include "qgsmapserviceexception.h" #include "qgsnetworkaccessmanager.h" +#include "qgsnetworkdiskcache.h" #include "qgsserverlogger.h" #include "qgsserverrequest.h" #include "qgsfilterresponsedecorator.h" @@ -88,9 +89,9 @@ void QgsServer::setupNetworkAccessManager() const QSettings settings; QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance(); QNetworkDiskCache *cache = new QNetworkDiskCache( nullptr ); - const qint64 cacheSize = sSettings()->cacheSize(); const QString cacheDirectory = sSettings()->cacheDirectory(); cache->setCacheDirectory( cacheDirectory ); + qint64 cacheSize = sSettings()->cacheSize(); cache->setMaximumCacheSize( cacheSize ); QgsMessageLog::logMessage( QStringLiteral( "cacheDirectory: %1" ).arg( cache->cacheDirectory() ), QStringLiteral( "Server" ), Qgis::MessageLevel::Info ); QgsMessageLog::logMessage( QStringLiteral( "maximumCacheSize: %1" ).arg( cache->maximumCacheSize() ), QStringLiteral( "Server" ), Qgis::MessageLevel::Info ); diff --git a/src/server/qgsserversettings.cpp b/src/server/qgsserversettings.cpp index 2b5729ec91b5..86e554f14d07 100644 --- a/src/server/qgsserversettings.cpp +++ b/src/server/qgsserversettings.cpp @@ -123,10 +123,10 @@ void QgsServerSettings::initSettings() // cache size const Setting sCacheSize = { QgsServerSettingsEnv::QGIS_SERVER_CACHE_SIZE, QgsServerSettingsEnv::DEFAULT_VALUE, - QStringLiteral( "Specify the cache size" ), - QStringLiteral( "/cache/size" ), - QVariant::LongLong, - QVariant( 256 * 1024 * 1024 ), + QStringLiteral( "Specify the cache size (0 = automatic size)" ), + QStringLiteral( "/cache/size-bytes" ), + QMetaType::Type::LongLong, + 0, QVariant() }; mSettings[ sCacheSize.envVar ] = sCacheSize; diff --git a/src/ui/qgsoptionsbase.ui b/src/ui/qgsoptionsbase.ui index 74b2bb1aa72e..95b9b650945a 100644 --- a/src/ui/qgsoptionsbase.ui +++ b/src/ui/qgsoptionsbase.ui @@ -4821,7 +4821,7 @@ The bigger the number, the faster zooming with the mouse wheel will be. - Size [KiB] + Size @@ -4836,7 +4836,32 @@ The bigger the number, the faster zooming with the mouse wheel will be. - + + + Specify the cache size in megabytes. Clear the value to enable smart cache size, which sets the maximum cache size based on available space. + + + MB + + + 0 + + + 10000 + + + 100 + + + 0 + + + true + + + Smart cache size + + diff --git a/tests/src/python/test_qgsserver_settings.py b/tests/src/python/test_qgsserver_settings.py index 4d77f033513f..00ab68bc8284 100644 --- a/tests/src/python/test_qgsserver_settings.py +++ b/tests/src/python/test_qgsserver_settings.py @@ -20,7 +20,7 @@ from utilities import unitTestDataPath -DEFAULT_CACHE_SIZE = 256 * 1024 * 1024 +DEFAULT_CACHE_SIZE = 0 # Smart cache size class TestQgsServerSettings(unittest.TestCase): diff --git a/tests/testdata/qgis_server_settings/conf0.ini b/tests/testdata/qgis_server_settings/conf0.ini index 011717bf364e..e6c5f8e15ea9 100644 --- a/tests/testdata/qgis_server_settings/conf0.ini +++ b/tests/testdata/qgis_server_settings/conf0.ini @@ -1,6 +1,6 @@ [cache] directory= -size=@Variant(\0\0\0\x81\0\0\0\0\x3 \0\0) +size-bytes=@Variant(\0\0\0\x81\0\0\0\0\x3 \0\0) [qgis] parallel_rendering=true diff --git a/tests/testdata/qgis_server_settings/conf1.ini b/tests/testdata/qgis_server_settings/conf1.ini index 6c40c580df52..c4f62f0b56d1 100644 --- a/tests/testdata/qgis_server_settings/conf1.ini +++ b/tests/testdata/qgis_server_settings/conf1.ini @@ -1,6 +1,6 @@ [cache] directory=/tmp/mycache -size=@Variant(\0\0\0\x81\0\0\0\0\x3 \0\0) +size-bytes=@Variant(\0\0\0\x81\0\0\0\0\x3 \0\0) [qgis] parallel_rendering=false