Add reference linking as an option for the post processor. (#3633)

* Add reference linking as an option for the post processor. * Fix linting issue * Fix some small formatting issues * Add note to remove .encode when https://goo.gl/BMn8EC is merged
pymedusa · Mar 7, 2018 · 729945d · 729945d
1 parent 29b963b
commit 729945d
Show file tree

Hide file tree

Showing 11 changed files with 192 additions and 40 deletions.
diff --git a/medusa/helpers/__init__.py b/medusa/helpers/__init__.py
@@ -67,6 +67,11 @@
 log = BraceAdapter(logging.getLogger(__name__))
 log.logger.addHandler(logging.NullHandler())
 
+try:
+    import reflink
+except ImportError:
+    reflink = None
+
 
 def indent_xml(elem, level=0):
     """Do our pretty printing and make Matt very happy."""
@@ -422,6 +427,68 @@ def move_and_symlink_file(src_file, dest_file):
             copy_file(src_file, dest_file)
 
 
+def reflink_file(src_file, dest_file):
+    """Copy a file from source to destination with a reference link.
+
+    :param src_file: Source file
+    :type src_file: str
+    :param dest_file: Destination file
+    :type dest_file: str
+    """
+    try:
+        if reflink is None:
+            raise NotImplementedError()
+        reflink.reflink(src_file.encode('utf-8'), dest_file.encode('utf-8'))  # NOTE: remove when https://goo.gl/BMn8EC is merged.
+    except reflink.ReflinkImpossibleError as msg:
+        if msg.args[0] == 'EOPNOTSUPP':
+            log.warning(
+                u'Failed to create reference link of {source} at {destination}.'
+                u' Error: Filesystem or OS has not implemented reflink. Copying instead', {
+                    'source': src_file,
+                    'destination': dest_file,
+                }
+            )
+            copy_file(src_file, dest_file)
+        elif msg.args[0] == 'EXDEV':
+            log.warning(
+                u'Failed to create reference link of {source} at {destination}.'
+                u' Error: Can not reflink between two devices. Copying instead', {
+                    'source': src_file,
+                    'destination': dest_file,
+                }
+            )
+            copy_file(src_file, dest_file)
+        else:
+            log.warning(
+                u'Failed to create reflink of {source} at {destination}.'
+                u' Error: {error!r}. Copying instead', {
+                    'source': src_file,
+                    'destination': dest_file,
+                    'error': msg,
+                }
+            )
+            copy_file(src_file, dest_file)
+    except NotImplementedError:
+        log.warning(
+            u'Failed to create reference link of {source} at {destination}.'
+            u' Error: Filesystem does not support reflink or reflink is not installed. Copying instead', {
+                'source': src_file,
+                'destination': dest_file,
+            }
+        )
+        copy_file(src_file, dest_file)
+    except IOError as msg:
+        log.warning(
+            u'Failed to create reflink of {source} at {destination}.'
+            u' Error: {error!r}. Copying instead', {
+                'source': src_file,
+                'destination': dest_file,
+                'error': msg,
+            }
+        )
+        copy_file(src_file, dest_file)
+
+
 def make_dirs(path):
     """Create any folders that are missing and assigns them the permissions of their parents.
 

diff --git a/themes-default/legacy/views/config_postProcessing.mako b/themes-default/legacy/views/config_postProcessing.mako
@@ -2,6 +2,7 @@
 <%!
     import os.path
     import datetime
+    import pkgutil
     from medusa import app
     from medusa.common import SKIPPED, WANTED, UNAIRED, ARCHIVED, IGNORED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, FAILED
     from medusa.common import Quality, qualityPresets, statusStrings, qualityPresetStrings, cpu_presets, MULTI_EP_STRINGS
@@ -62,10 +63,17 @@
                                     <span class="component-title">Processing Method:</span>
                                     <span class="component-desc">
                                         <select name="process_method" id="process_method" class="form-control input-sm">
-                                            <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
-                                            % for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
-                                            <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
-                                            % endfor
+                                            % if pkgutil.find_loader('reflink') is not None:
+                                                <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
+                                                % for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
+                                                    <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                                                % endfor
+                                            % else:
+                                                % for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
+                                                    <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                                                % endfor
+                                                <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
+                                            % endif
                                         </select>
                                     </span>
                                 </label>
@@ -76,6 +84,7 @@
                                 <label class="nocheck">
                                     <span class="component-title">&nbsp;</span>
                                     <span class="component-desc"><b>NOTE:</b> If you keep seeding torrents after they finish, please avoid the 'move' processing method to prevent errors.</span>
+                                    <span class="component-desc">To use reference linking, the <a href="http://www.dereferer.org/?https://pypi.python.org/pypi/reflink/0.1.4">reflink package</a> needs to be installed.</span>
                                 </label>
                             </div>
                             <div class="field-pair">

diff --git a/themes-default/legacy/views/home_postprocess.mako b/themes-default/legacy/views/home_postprocess.mako
@@ -1,5 +1,6 @@
 <%inherit file="/layouts/main.mako"/>
 <%!
+    import pkgutil
     from medusa import app
 %>
 <%block name="content">
@@ -32,10 +33,17 @@
                 </td>
                 <td>
                     <select name="process_method" id="process_method" class="form-control form-control-inline input-sm" >
-                    <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
-                    % for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
-                        <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
-                    % endfor
+                    % if pkgutil.find_loader('reflink') is not None:
+                        <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
+                        % for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
+                            <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                        % endfor
+                    % else:
+                        <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
+                        % for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
+                            <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                        % endfor
+                    % endif
                     </select>
                 </td>
             </tr>

diff --git a/themes-default/slim/views/config_postProcessing.mako b/themes-default/slim/views/config_postProcessing.mako
@@ -2,6 +2,7 @@
 <%!
     import os.path
     import datetime
+    import pkgutil
     from medusa import app
     from medusa.common import SKIPPED, WANTED, UNAIRED, ARCHIVED, IGNORED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, FAILED
     from medusa.common import Quality, qualityPresets, statusStrings, qualityPresetStrings, cpu_presets, MULTI_EP_STRINGS
@@ -62,10 +63,17 @@
                                     <span class="component-title">Processing Method:</span>
                                     <span class="component-desc">
                                         <select name="process_method" id="process_method" class="form-control input-sm">
-                                            <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
-                                            % for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
-                                            <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
-                                            % endfor
+                                            % if pkgutil.find_loader('reflink') is not None:
+                                                <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
+                                                % for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
+                                                    <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                                                % endfor
+                                            % else:
+                                                % for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
+                                                    <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                                                % endfor
+                                                <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
+                                            % endif
                                         </select>
                                     </span>
                                 </label>
@@ -76,6 +84,7 @@
                                 <label class="nocheck">
                                     <span class="component-title">&nbsp;</span>
                                     <span class="component-desc"><b>NOTE:</b> If you keep seeding torrents after they finish, please avoid the 'move' processing method to prevent errors.</span>
+                                    <span class="component-desc">To use reference linking, the <a href="http://www.dereferer.org/?https://pypi.python.org/pypi/reflink/0.1.4">reflink package</a> needs to be installed.</span>
                                 </label>
                             </div>
                             <div class="field-pair">

diff --git a/themes-default/slim/views/home_postprocess.mako b/themes-default/slim/views/home_postprocess.mako
@@ -1,5 +1,6 @@
 <%inherit file="/layouts/main.mako"/>
 <%!
+    import pkgutil
     from medusa import app
 %>
 <%block name="content">
@@ -32,10 +33,17 @@
                 </td>
                 <td>
                     <select name="process_method" id="process_method" class="form-control form-control-inline input-sm" >
-                    <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
-                    % for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
-                        <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
-                    % endfor
+                    % if pkgutil.find_loader('reflink') is not None:
+                        <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
+                        % for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
+                            <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                        % endfor
+                    % else:
+                        <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
+                        % for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
+                            <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                        % endfor
+                    % endif
                     </select>
                 </td>
             </tr>

diff --git a/themes/dark/templates/config_postProcessing.mako b/themes/dark/templates/config_postProcessing.mako
@@ -2,6 +2,7 @@
 <%!
     import os.path
     import datetime
+    import pkgutil
     from medusa import app
     from medusa.common import SKIPPED, WANTED, UNAIRED, ARCHIVED, IGNORED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, FAILED
     from medusa.common import Quality, qualityPresets, statusStrings, qualityPresetStrings, cpu_presets, MULTI_EP_STRINGS
@@ -62,10 +63,17 @@
                                     <span class="component-title">Processing Method:</span>
                                     <span class="component-desc">
                                         <select name="process_method" id="process_method" class="form-control input-sm">
-                                            <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
-                                            % for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
-                                            <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
-                                            % endfor
+                                            % if pkgutil.find_loader('reflink') is not None:
+                                                <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
+                                                % for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
+                                                    <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                                                % endfor
+                                            % else:
+                                                % for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
+                                                    <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                                                % endfor
+                                                <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
+                                            % endif
                                         </select>
                                     </span>
                                 </label>
@@ -76,6 +84,7 @@
                                 <label class="nocheck">
                                     <span class="component-title">&nbsp;</span>
                                     <span class="component-desc"><b>NOTE:</b> If you keep seeding torrents after they finish, please avoid the 'move' processing method to prevent errors.</span>
+                                    <span class="component-desc">To use reference linking, the <a href="http://www.dereferer.org/?https://pypi.python.org/pypi/reflink/0.1.4">reflink package</a> needs to be installed.</span>
                                 </label>
                             </div>
                             <div class="field-pair">

diff --git a/themes/dark/templates/home_postprocess.mako b/themes/dark/templates/home_postprocess.mako
@@ -1,5 +1,6 @@
 <%inherit file="/layouts/main.mako"/>
 <%!
+    import pkgutil
     from medusa import app
 %>
 <%block name="content">
@@ -32,10 +33,17 @@
                 </td>
                 <td>
                     <select name="process_method" id="process_method" class="form-control form-control-inline input-sm" >
-                    <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
-                    % for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
-                        <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
-                    % endfor
+                    % if pkgutil.find_loader('reflink') is not None:
+                        <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
+                        % for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
+                            <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                        % endfor
+                    % else:
+                        <% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
+                        % for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
+                            <option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
+                        % endfor
+                    % endif
                     </select>
                 </td>
             </tr>