diff --git a/holoviews/core/data/pandas.py b/holoviews/core/data/pandas.py
index b97535d35e..2292e084b7 100644
--- a/holoviews/core/data/pandas.py
+++ b/holoviews/core/data/pandas.py
@@ -361,9 +361,9 @@ def sorted_index(cls, df):
     def sort_depth(cls, df):
         try:
             from pandas.core.indexes.multi import _lexsort_depth
-        except Exception:
+            return _lexsort_depth(df.index.codes, df.index.nlevels)
+        except (ImportError, AttributeError):
             return 0
-        return _lexsort_depth(df.index.codes, df.index.nlevels)
 
     @classmethod
     def index_selection(cls, df, selection):