diff --git a/asv_bench/benchmarks/benchmarks.py b/asv_bench/benchmarks/benchmarks.py
index 281b18ca924..c128047d797 100644
--- a/asv_bench/benchmarks/benchmarks.py
+++ b/asv_bench/benchmarks/benchmarks.py
@@ -21,12 +21,20 @@
 import numpy as np
 import pandas
 
-from modin.config import TestDatasetSize, AsvImplementation
 from .utils import generate_dataframe, RAND_LOW, RAND_HIGH, random_string
 
-ASV_USE_IMPL = AsvImplementation.get()
+try:
+    from modin.config import TestDatasetSize, AsvImplementation
 
-if TestDatasetSize.get() == "Big":
+    ASV_USE_IMPL = AsvImplementation.get()
+    ASV_DATASET_SIZE = TestDatasetSize.get()
+except ImportError:
+    # The same benchmarking code can be run for different versions of Modin, so in
+    # case of an error importing important variables, we'll just use predefined values
+    ASV_USE_IMPL = "modin"
+    ASV_DATASET_SIZE = "Big" if pd.DEFAULT_NPARTITIONS >= 32 else "Small"
+
+if ASV_DATASET_SIZE == "Big":
     BINARY_OP_DATA_SIZE = [
         (5000, 5000, 5000, 5000),
         # the case extremely inefficient