diff --git a/autosklearn/automl.py b/autosklearn/automl.py
index b5e8a9becd..414a0ee8d8 100644
--- a/autosklearn/automl.py
+++ b/autosklearn/automl.py
@@ -1268,8 +1268,8 @@ def cv_results_(self):
 
         results['mean_fit_time'] = np.array(mean_fit_time)
         results['params'] = params
-        results['rank_test_scores'] = scipy.stats.rankdata(1 - results['mean_test_score'],
-                                                           method='min')
+        rank_order = -1 * self._metric._sign * results['mean_test_score']
+        results['rank_test_scores'] = scipy.stats.rankdata(rank_order, method='min')
         results['status'] = status
         results['budgets'] = budgets