Fix throttling by using youtube-dl downloader

pafy/backend_shared.py

@@ -563,8 +563,8 @@ def cancel(self):
             self._active = False
             return True
 
-    def download(self, filepath="", quiet=False, progress="Bytes", callback=None,
-                 meta=False, remux_audio=False):
+    def download(self, filepath="", quiet=False, progress="Bytes",
+                           callback=None, meta=False, remux_audio=False):
         """ Download.  Use quiet=True to supress output. Return filename.
 
         Use meta=True to append video id and itag to generated filename
@@ -588,15 +588,10 @@ def download(self, filepath="", quiet=False, progress="Bytes", callback=None,
         temp_filepath = filepath + ".temp"
 
         progress_available = ["KB", "MB", "GB"]
-        if not progress in progress_available:
+        if progress not in progress_available:
             progress = "Bytes"
 
-        status_string = ('  {:,} ' + progress + ' [{:.2%}] received. Rate: [{:4.0f} '
-                         'KB/s].  ETA: [{:.0f} secs]')
-
-        if early_py_version:
-            status_string = ('  {0:} ' + progress + ' [{1:.2%}] received. Rate:'
-                             ' [{2:4.0f} KB/s].  ETA: [{3:.0f} secs]')
+        status_string = get_status_string(progress)
 
         response = g.opener.open(self.url)
         total = int(response.info()['Content-Length'].strip())
@@ -630,18 +625,12 @@ def download(self, filepath="", quiet=False, progress="Bytes", callback=None,
             if elapsed:
                 rate = ((float(bytesdone) - float(offset)) / 1024.0) / elapsed
                 eta = (total - bytesdone) / (rate * 1024)
-            else: # Avoid ZeroDivisionError
+            else:  # Avoid ZeroDivisionError
                 rate = 0
                 eta = 0
 
-            if progress == "KB":
-                progress_stats = (round(bytesdone/1024.0, 2), bytesdone * 1.0 / total, rate, eta)
-            elif progress == "MB":
-                progress_stats = (round(bytesdone/1048576.0, 2), bytesdone * 1.0 / total, rate, eta)
-            elif progress == "GB":
-                progress_stats = (round(bytesdone/1073741824.0, 2), bytesdone * 1.0 / total, rate, eta)
-            else:
-                progress_stats = (bytesdone, bytesdone * 1.0 / total, rate, eta)
+            progress_stats = (get_size_done(bytesdone, progress),
+                              bytesdone * 1.0 / total, rate, eta)
 
             if not chunk:
                 outfh.close()
@@ -696,3 +685,19 @@ def remux(infile, outfile, quiet=False, muxer="ffmpeg"):
     else:
         logging.warning("audio remux failed")
         os.rename(infile, outfile)
+
+
+def get_size_done(bytesdone, progress):
+    _progress_dict = {'KB': 1024.0, 'MB': 1048576.0, 'GB': 1073741824.0}
+    return round(bytesdone/_progress_dict.get(progress, 1.0), 2)
+
+
+def get_status_string(progress):
+    status_string = ('  {:,} ' + progress + ' [{:.2%}] received. Rate: [{:4.0f} '
+                     'KB/s].  ETA: [{:.0f} secs]')
+
+    if early_py_version:
+        status_string = ('  {0:} ' + progress + ' [{1:.2%}] received. Rate:'
+                         ' [{2:4.0f} KB/s].  ETA: [{3:.0f} secs]')
+
+    return status_string

pafy/backend_youtube_dl.py

@@ -1,6 +1,8 @@
 import sys
 import time
 import logging
+import os
+import subprocess
 
 if sys.version_info[:2] >= (3, 0):
     # pylint: disable=E0611,F0401,I0011
@@ -11,7 +13,7 @@
 import youtube_dl
 
 from . import g
-from .backend_shared import BasePafy, BaseStream, remux
+from .backend_shared import BasePafy, BaseStream, remux, get_status_string, get_size_done
 
 dbg = logging.debug
 
@@ -125,3 +127,79 @@ def get_filesize(self):
 
         # Fallback
         return super(YtdlStream, self).get_filesize()
+
+    def download(self, filepath="", quiet=False, progress="Bytes",
+                 callback=None, meta=False, remux_audio=False):
+
+        downloader = youtube_dl.downloader.http.HttpFD(ydl(),
+            {'http_chunk_size': 10485760})
+
+        progress_available = ["KB", "MB", "GB"]
+        if progress not in progress_available:
+            progress = "Bytes"
+
+        status_string = get_status_string(progress)
+
+        def progress_hook(s):
+            if s['status'] == 'downloading':
+                bytesdone = s['downloaded_bytes']
+                total = s['total_bytes']
+                if s.get('speed') is not None:
+                    rate = s['speed'] / 1024
+                else:
+                    rate = 0
+                if s.get('eta') is None:
+                    eta = 0
+                else:
+                    eta = s['eta']
+
+                progress_stats = (get_size_done(bytesdone, progress),
+                                  bytesdone*1.0/total, rate, eta)
+                if not quiet:
+                    status = status_string.format(*progress_stats)
+                    sys.stdout.write("\r" + status + ' ' * 4 + "\r")
+                    sys.stdout.flush()
+
+                if callback:
+                    callback(total, *progress_stats)
+
+        downloader._progress_hooks = [progress_hook]
+
+        if filepath and os.path.isdir(filepath):
+            filename = self.generate_filename(max_length=256 - len('.temp'))
+            filepath = os.path.join(filepath, filename)
+
+        elif filepath:
+            pass
+
+        else:
+            filepath = self.generate_filename(meta=meta, max_length=256 - len('.temp'))
+
+        infodict = {'url': self.url}
+
+        downloader.download(filepath, infodict)
+        print()
+
+        if remux_audio and self.mediatype == "audio":
+            subprocess.run(['mv', filepath, filepath + '.temp'])
+            remux(filepath + '.temp', filepath, quiet=quiet, muxer=remux_audio)
+
+
+class ydl:
+    def urlopen(self, url):
+        return g.opener.open(url)
+
+    def to_screen(self, *args, **kwargs):
+        pass
+
+    def to_console_title(self, *args, **kwargs):
+        pass
+
+    def trouble(self, *args, **kwargs):
+        pass
+
+    def report_warning(self, *args, **kwargs):
+        pass
+
+    def report_error(self, *args, **kwargs):
+        pass