diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp
index f51a411e82..d938118036 100644
--- a/cpp/include/kvikio/file_handle.hpp
+++ b/cpp/include/kvikio/file_handle.hpp
@@ -165,13 +165,17 @@ class FileHandle {
       _compat_mode{compat_mode}
   {
     if (!_compat_mode) {
-      try {
-        _fd_direct_on = detail::open_fd(file_path, flags, true, mode);
-      } catch (const std::system_error&) {
-        _compat_mode = true;  // Fall back to compat mode if we cannot open the file with O_DIRECT
-      }
+      return;  // No need to open the file with the O_DIRECT flag in compatibility mode
     }
 
+    // Try to open the file with the O_DIRECT flag. Fall back to compatibility mode, if it fails.
+    try {
+      _fd_direct_on = detail::open_fd(file_path, flags, true, mode);
+    } catch (const std::system_error&) {
+      _compat_mode = true;
+    }
+
+    // Create a cuFile handle, if not in compatibility mode
     if (!_compat_mode) {
       CUfileDescr_t desc{};  // It is important to set to zero!
       desc.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD;