Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add reference linking as an option for the post processor. #3633

Merged
merged 6 commits into from
Mar 7, 2018
Merged

Add reference linking as an option for the post processor. #3633

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4f0dbb5
Add reference linking as an option for the post processor.
Jan 19, 2018
884304a
Rebase
Kriskras99 Mar 7, 2018
ef8e298
Merge branch 'pymedusa-develop' into feature/reflink
Kriskras99 Mar 7, 2018
12a93eb
Fix linting issue
Kriskras99 Mar 7, 2018
49bcaa1
Fix some small formatting issues
Kriskras99 Mar 7, 2018
c21f3a2
Add note to remove .encode when https://goo.gl/BMn8EC is merged
Kriskras99 Mar 7, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions medusa/helpers/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@
log = BraceAdapter(logging.getLogger(__name__))
log.logger.addHandler(logging.NullHandler())

try:
import reflink
except ImportError:
reflink = None


def indent_xml(elem, level=0):
"""Do our pretty printing and make Matt very happy."""
Expand Down Expand Up @@ -422,6 +427,68 @@ def move_and_symlink_file(src_file, dest_file):
copy_file(src_file, dest_file)


def reflink_file(src_file, dest_file):
"""Copy a file from source to destination with a reference link.

:param src_file: Source file
:type src_file: str
:param dest_file: Destination file
:type dest_file: str
"""
try:
if reflink is None:
raise NotImplementedError()
reflink.reflink(src_file.encode('utf-8'), dest_file.encode('utf-8'))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the encoding of src_file and dest_file really needed here? I've noticed that the reflink package supports Python 3 as well, so Unicode paths shouldn't be an issue.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's because otherwise reflink will encode any string as ascii, which will fail on some foreign shows.
See this pull request (which I just opened because I forgot about this).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see what you mean. In this case, can you please add a comment referencing the issue?

except reflink.ReflinkImpossibleError as msg:
if msg.args[0] == 'EOPNOTSUPP':
log.warning(
u'Failed to create reference link of {source} at {destination}.'
u' Error: Filesystem or OS has not implemented reflink. Copying instead', {
'source': src_file,
'destination': dest_file,
}
)
copy_file(src_file, dest_file)
elif msg.args[0] == 'EXDEV':
log.warning(
u'Failed to create reference link of {source} at {destination}.'
u' Error: Can not reflink between two devices. Copying instead', {
'source': src_file,
'destination': dest_file,
}
)
copy_file(src_file, dest_file)
else:
log.warning(
u'Failed to create reflink of {source} at {destination}.'
u' Error: {error!r}. Copying instead', {
'source': src_file,
'destination': dest_file,
'error': msg,
}
)
copy_file(src_file, dest_file)
except NotImplementedError:
log.warning(
u'Failed to create reference link of {source} at {destination}.'
u' Error: Filesystem does not support reflink or reflink is not installed. Copying instead', {
'source': src_file,
'destination': dest_file,
}
)
copy_file(src_file, dest_file)
except IOError as msg:
log.warning(
u'Failed to create reflink of {source} at {destination}.'
u' Error: {error!r}. Copying instead', {
'source': src_file,
'destination': dest_file,
'error': msg,
}
)
copy_file(src_file, dest_file)


def make_dirs(path):
"""Create any folders that are missing and assigns them the permissions of their parents.

Expand Down
17 changes: 13 additions & 4 deletions themes-default/legacy/views/config_postProcessing.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<%!
import os.path
import datetime
import pkgutil
from medusa import app
from medusa.common import SKIPPED, WANTED, UNAIRED, ARCHIVED, IGNORED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, FAILED
from medusa.common import Quality, qualityPresets, statusStrings, qualityPresetStrings, cpu_presets, MULTI_EP_STRINGS
Expand Down Expand Up @@ -62,10 +63,17 @@
<span class="component-title">Processing Method:</span>
<span class="component-desc">
<select name="process_method" id="process_method" class="form-control input-sm">
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% if pkgutil.find_loader("reflink") is not None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use single quotes here.

<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% else:
% for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% endif
</select>
</span>
</label>
Expand All @@ -76,6 +84,7 @@
<label class="nocheck">
<span class="component-title">&nbsp;</span>
<span class="component-desc"><b>NOTE:</b> If you keep seeding torrents after they finish, please avoid the 'move' processing method to prevent errors.</span>
<span class="component-desc">To use reference linking, the <a href="http://www.dereferer.org/?https://pypi.python.org/pypi/reflink/0.1.4">reflink package</a> needs to be installed.</span>
</label>
</div>
<div class="field-pair">
Expand Down
16 changes: 12 additions & 4 deletions themes-default/legacy/views/home_postprocess.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
<%inherit file="/layouts/main.mako"/>
<%!
import pkgutil
from medusa import app
%>
<%block name="content">
Expand Down Expand Up @@ -32,10 +33,17 @@
</td>
<td>
<select name="process_method" id="process_method" class="form-control form-control-inline input-sm" >
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% if pkgutil.find_loader("reflink") is not None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use single quotes here.

<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% else:
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% endif
</select>
</td>
</tr>
Expand Down
18 changes: 14 additions & 4 deletions themes-default/slim/views/config_postProcessing.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<%!
import os.path
import datetime
import pkgutil
from medusa import app
from medusa.common import SKIPPED, WANTED, UNAIRED, ARCHIVED, IGNORED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, FAILED
from medusa.common import Quality, qualityPresets, statusStrings, qualityPresetStrings, cpu_presets, MULTI_EP_STRINGS
Expand Down Expand Up @@ -62,10 +63,17 @@
<span class="component-title">Processing Method:</span>
<span class="component-desc">
<select name="process_method" id="process_method" class="form-control input-sm">
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% if pkgutil.find_loader("reflink") is not None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use single quotes here.

<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% else:
% for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% endif
</select>
</span>
</label>
Expand All @@ -76,6 +84,8 @@
<label class="nocheck">
<span class="component-title">&nbsp;</span>
<span class="component-desc"><b>NOTE:</b> If you keep seeding torrents after they finish, please avoid the 'move' processing method to prevent errors.</span>

<span class="component-desc">To use reference linking, the <a href="http://www.dereferer.org/?https://pypi.python.org/pypi/reflink/0.1.4">reflink package</a> needs to be installed.</span>
</label>
</div>
<div class="field-pair">
Expand Down
16 changes: 12 additions & 4 deletions themes-default/slim/views/home_postprocess.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
<%inherit file="/layouts/main.mako"/>
<%!
import pkgutil
from medusa import app
%>
<%block name="content">
Expand Down Expand Up @@ -32,10 +33,17 @@
</td>
<td>
<select name="process_method" id="process_method" class="form-control form-control-inline input-sm" >
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% if pkgutil.find_loader("reflink") is not None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use single quotes here.

<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% else:
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% endif
</select>
</td>
</tr>
Expand Down
18 changes: 14 additions & 4 deletions themes/dark/templates/config_postProcessing.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<%!
import os.path
import datetime
import pkgutil
from medusa import app
from medusa.common import SKIPPED, WANTED, UNAIRED, ARCHIVED, IGNORED, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, FAILED
from medusa.common import Quality, qualityPresets, statusStrings, qualityPresetStrings, cpu_presets, MULTI_EP_STRINGS
Expand Down Expand Up @@ -62,10 +63,17 @@
<span class="component-title">Processing Method:</span>
<span class="component-desc">
<select name="process_method" id="process_method" class="form-control input-sm">
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% if pkgutil.find_loader("reflink") is not None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use single quotes here.

<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% else:
% for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% endif
</select>
</span>
</label>
Expand All @@ -76,6 +84,8 @@
<label class="nocheck">
<span class="component-title">&nbsp;</span>
<span class="component-desc"><b>NOTE:</b> If you keep seeding torrents after they finish, please avoid the 'move' processing method to prevent errors.</span>

<span class="component-desc">To use reference linking, the <a href="http://www.dereferer.org/?https://pypi.python.org/pypi/reflink/0.1.4">reflink package</a> needs to be installed.</span>
</label>
</div>
<div class="field-pair">
Expand Down
16 changes: 12 additions & 4 deletions themes/dark/templates/home_postprocess.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
<%inherit file="/layouts/main.mako"/>
<%!
import pkgutil
from medusa import app
%>
<%block name="content">
Expand Down Expand Up @@ -32,10 +33,17 @@
</td>
<td>
<select name="process_method" id="process_method" class="form-control form-control-inline input-sm" >
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% if pkgutil.find_loader("reflink") is not None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use single quotes here.

<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link", 'reflink': "Reference Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink', 'reflink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% else:
<% process_method_text = {'copy': "Copy", 'move': "Move", 'hardlink': "Hard Link", 'symlink' : "Symbolic Link"} %>
% for cur_action in ('copy', 'move', 'hardlink', 'symlink'):
<option value="${cur_action}" ${'selected="selected"' if app.PROCESS_METHOD == cur_action else ''}>${process_method_text[cur_action]}</option>
% endfor
% endif
</select>
</td>
</tr>
Expand Down
Loading