diff --git a/src/Flysystem/AssetAdapter.php b/src/Flysystem/AssetAdapter.php
index d6c1d3fd..a6cabb3f 100644
--- a/src/Flysystem/AssetAdapter.php
+++ b/src/Flysystem/AssetAdapter.php
@@ -146,8 +146,20 @@ protected function configureServer($forceOverwrite = false)
 
         // Apply each configuration
         $config = new FlysystemConfig();
-        $config->set('visibility', $visibility);
         foreach ($configurations as $file => $template) {
+            $perms = $this->getVisibility($file);
+
+            if ($this->has($file)) {
+                if ($perms['visibility'] === $visibility) {
+                    // If the visibility is already correct, do not try to re-set it.
+                    // Attempting to set the visibility of a file/folder when the filesystem is already
+                    // set correctly causes "Operation not permitted" errors on some Linux servers.
+                    $config->set('visibility', null);
+                } else {
+                    $config->set('visibility', $visibility);
+                }
+            }
+
             // Ensure file contents
             if ($forceOverwrite || !$this->has($file)) {
                 // Evaluate file
@@ -157,7 +169,6 @@ protected function configureServer($forceOverwrite = false)
                     throw new Exception("Error writing server configuration file \"{$file}\"");
                 }
             }
-            $perms = $this->getVisibility($file);
             if ($perms['visibility'] !== $visibility) {
                 // Ensure correct permissions
                 $this->setVisibility($file, $visibility);