diff --git a/src/mixer/playermanager.cpp b/src/mixer/playermanager.cpp
index c8d162a58117..d1a6d00bd455 100644
--- a/src/mixer/playermanager.cpp
+++ b/src/mixer/playermanager.cpp
@@ -450,7 +450,7 @@ void PlayerManager::addDeckInner() {
 }
 
 void PlayerManager::loadSamplers() {
-    m_pSamplerBank->loadSamplerBankFromPath(getDefaultSamplerPath(m_pConfig));
+    m_pSamplerBank->loadSamplerBankFromPath(getDefaultSamplerPath(m_pConfig), true);
 }
 
 void PlayerManager::addSampler() {
diff --git a/src/mixer/samplerbank.cpp b/src/mixer/samplerbank.cpp
index 267ad2183218..1c9f71bfe3a0 100644
--- a/src/mixer/samplerbank.cpp
+++ b/src/mixer/samplerbank.cpp
@@ -165,7 +165,7 @@ void SamplerBank::slotLoadSamplerBank(double v) {
     }
 }
 
-bool SamplerBank::loadSamplerBankFromPath(const QString& samplerBankPath) {
+bool SamplerBank::loadSamplerBankFromPath(const QString& samplerBankPath, bool initializing) {
     // The user has picked a new directory via a file dialog. This means the
     // system sandboxer (if we are sandboxed) has granted us permission to this
     // folder. We don't need access to this file on a regular basis so we do not
@@ -195,33 +195,48 @@ bool SamplerBank::loadSamplerBankFromPath(const QString& samplerBankPath) {
         return false;
     }
 
-    QDomNode n = root.firstChild();
-
-    while (!n.isNull()) {
+    const auto samplerNodes = root.childNodes();
+    if (samplerNodes.isEmpty()) {
+        return true;
+    }
+    for (int i = 0; i < samplerNodes.size(); i++) {
+        QDomNode n = samplerNodes.at(i);
         QDomElement e = n.toElement();
+        if (e.isNull() || e.tagName() != "sampler") {
+            continue;
+        }
+
+        const QString group = e.attribute("group", "");
+        const QString location = e.attribute("location", "");
+        int samplerNum;
+
+        if (group.isEmpty() || m_pPlayerManager->isSamplerGroup(group, &samplerNum)) {
+            continue;
+        }
+
+        // During startup we only increase the sampler count if there is
+        // a track to be loaded. This avoids
+        // * creating more (unneedded) sampler players than engine, skin or
+        //   some controller mapping requested
+        // * an unnecessary large Load To > Sampler N submenu in the track menu
+        if (initializing && location.isEmpty()) {
+            continue;
+        }
+
+        // Later on, when the user loads a samplers file manually, we
+        // want to eject loaded tracks if the file's sample slot is empty.
+        // We also create new players even if they are not present in the
+        // GUI to not drop tracks loaded to invisible samplers when saving
+        // the samplers file.
+        if (m_pPlayerManager->numSamplers() < (unsigned)samplerNum) {
+            m_pCONumSamplers->set(samplerNum);
+        }
 
-        if (!e.isNull()) {
-            if (e.tagName() == "sampler") {
-                QString group = e.attribute("group", "");
-                QString location = e.attribute("location", "");
-                int samplerNum;
-
-                if (!group.isEmpty()
-                        && m_pPlayerManager->isSamplerGroup(group, &samplerNum)) {
-                    if (m_pPlayerManager->numSamplers() < (unsigned) samplerNum) {
-                        m_pCONumSamplers->set(samplerNum);
-                    }
-
-                    if (location.isEmpty()) {
-                        m_pPlayerManager->slotLoadTrackToPlayer(TrackPointer(), group, false);
-                    } else {
-                        m_pPlayerManager->slotLoadLocationToPlayer(location, group, false);
-                    }
-                }
-
-            }
+        if (location.isEmpty()) {
+            m_pPlayerManager->slotLoadTrackToPlayer(TrackPointer(), group, false);
+        } else {
+            m_pPlayerManager->slotLoadLocationToPlayer(location, group, false);
         }
-        n = n.nextSibling();
     }
 
     file.close();
diff --git a/src/mixer/samplerbank.h b/src/mixer/samplerbank.h
index 5234f045f711..ee2f29c3456d 100644
--- a/src/mixer/samplerbank.h
+++ b/src/mixer/samplerbank.h
@@ -19,7 +19,7 @@ class SamplerBank : public QObject {
             PlayerManager* pPlayerManager);
 
     bool saveSamplerBankToPath(const QString& samplerBankPath);
-    bool loadSamplerBankFromPath(const QString& samplerBankPath);
+    bool loadSamplerBankFromPath(const QString& samplerBankPath, bool initializing = false);
 
   private slots:
     void slotSaveSamplerBank(double v);