Streaming pager: initial commit. Fixes pallets#409

The streaming pager works using generators.

Just pass a generator function or object into echo_via_pager:

generator expression:
```
click.echo_via_pager(("{}\n".format(i) for i in range(999999)))
```

generator function / expression:
```
def gen():
    counter = 0
    while True:
        counter++
        yield counter
click.echo_via_pager(gen)
click.echo_via_pager(gen()) # you can pass both
```

click/_termui_impl.py

@@ -13,9 +13,10 @@
 import sys
 import time
 import math
+import inspect
 from ._compat import _default_text_stdout, range_type, PY2, isatty, \
      open_stream, strip_ansi, term_len, get_best_encoding, WIN, int_types, \
-     CYGWIN
+     string_types, text_type, CYGWIN
 from .utils import echo
 from .exceptions import ClickException
 
@@ -272,35 +273,47 @@ def next(self):
         del next
 
 
-def pager(text, color=None):
+def pager(generator, color=None):
     """Decide what method to use for paging through text."""
     stdout = _default_text_stdout()
     if not isatty(sys.stdin) or not isatty(stdout):
-        return _nullpager(stdout, text, color)
+        return _nullpager(stdout, generator, color)
     pager_cmd = (os.environ.get('PAGER', None) or '').strip()
     if pager_cmd:
         if WIN:
-            return _tempfilepager(text, pager_cmd, color)
-        return _pipepager(text, pager_cmd, color)
+            return _tempfilepager(generator, pager_cmd, color)
+        return _pipepager(generator, pager_cmd, color)
     if os.environ.get('TERM') in ('dumb', 'emacs'):
-        return _nullpager(stdout, text, color)
+        return _nullpager(stdout, generator, color)
     if WIN or sys.platform.startswith('os2'):
-        return _tempfilepager(text, 'more <', color)
+        return _tempfilepager(generator, 'more <', color)
     if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0:
-        return _pipepager(text, 'less', color)
+        return _pipepager(generator, 'less', color)
 
     import tempfile
     fd, filename = tempfile.mkstemp()
     os.close(fd)
     try:
         if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0:
-            return _pipepager(text, 'more', color)
-        return _nullpager(stdout, text, color)
+            return _pipepager(generator, 'more', color)
+        return _nullpager(stdout, generator, color)
     finally:
         os.unlink(filename)
 
 
-def _pipepager(text, cmd, color):
+def text_generator(iterable, append=None):
+    """Convert any iterable to a text generator"""
+    for o in iterable:
+        if not isinstance(o, string_types):
+            yield text_type(o)
+        else:
+            yield o
+
+    if append:
+        yield append
+
+
+def _pipepager(generator, cmd, color):
     """Page through text by feeding it to another program.  Invoking a
     pager through this might support colors.
     """
@@ -318,17 +331,19 @@ def _pipepager(text, cmd, color):
         elif 'r' in less_flags or 'R' in less_flags:
             color = True
 
-    if not color:
-        text = strip_ansi(text)
-
     c = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
                          env=env)
     encoding = get_best_encoding(c.stdin)
     try:
-        c.stdin.write(text.encode(encoding, 'replace'))
-        c.stdin.close()
+        for text in generator:
+            if not color:
+                text = strip_ansi(text)
+
+            c.stdin.write(text.encode(encoding, 'replace'))
     except (IOError, KeyboardInterrupt):
         pass
+    else:
+        c.stdin.close()
 
     # Less doesn't respect ^C, but catches it for its own UI purposes (aborting
     # search or other commands inside less).
@@ -347,10 +362,12 @@ def _pipepager(text, cmd, color):
             break
 
 
-def _tempfilepager(text, cmd, color):
+def _tempfilepager(generator, cmd, color):
     """Page through text by invoking a program on a temporary file."""
     import tempfile
     filename = tempfile.mktemp()
+    # TODO: This never terminates if the passed generator never terminates.
+    text = "".join(generator)
     if not color:
         text = strip_ansi(text)
     encoding = get_best_encoding(sys.stdout)
@@ -362,11 +379,12 @@ def _tempfilepager(text, cmd, color):
         os.unlink(filename)
 
 
-def _nullpager(stream, text, color):
+def _nullpager(stream, generator, color):
     """Simply print unformatted text.  This is the ultimate fallback."""
-    if not color:
-        text = strip_ansi(text)
-    stream.write(text)
+    for text in generator:
+        if not color:
+            text = strip_ansi(text)
+        stream.write(text)
 
 
 class Editor(object):

click/termui.py

@@ -1,8 +1,9 @@
 import os
 import sys
 import struct
+import inspect
 
-from ._compat import raw_input, text_type, string_types, \
+from ._compat import raw_input, string_types, \
      isatty, strip_ansi, get_winterm_size, DEFAULT_COLUMNS, WIN
 from .utils import echo
 from .exceptions import Abort, UsageError
@@ -203,22 +204,29 @@ def ioctl_gwinsz(fd):
     return int(cr[1]), int(cr[0])
 
 
-def echo_via_pager(text, color=None):
+def echo_via_pager(text_or_generator, color=None):
     """This function takes a text and shows it via an environment specific
     pager on stdout.
 
     .. versionchanged:: 3.0
        Added the `color` flag.
 
-    :param text: the text to page.
+    :param text_or_generator: the text to page, or alternatively, a
+                              generator emitting the text to page.
     :param color: controls if the pager supports ANSI colors or not.  The
                   default is autodetection.
     """
     color = resolve_color_default(color)
-    if not isinstance(text, string_types):
-        text = text_type(text)
-    from ._termui_impl import pager
-    return pager(text + '\n', color)
+
+    if inspect.isgeneratorfunction(text_or_generator):
+        i = text_or_generator()
+    elif isinstance(text_or_generator, str):
+        i = [text_or_generator]
+    else:
+        i = iter(text_or_generator)
+
+    from ._termui_impl import pager, text_generator
+    return pager(text_generator(i, append='\n'), color)
 
 
 def progressbar(iterable=None, length=None, label=None, show_eta=True,

tests/test_utils.py

@@ -1,5 +1,7 @@
 import os
 import sys
+import time
+import itertools
 
 import pytest
 
@@ -146,14 +148,36 @@ def f(_):
     assert out == 'Password: \nScrew you.\n'
 
 
+def _test_gen_func():
+    yield 'a'
+    yield 'b'
+    yield 'c'
+    yield 'abc'
+
+
 @pytest.mark.skipif(WIN, reason='Different behavior on windows.')
 @pytest.mark.parametrize('cat', ['cat', 'cat ', 'cat '])
-def test_echo_via_pager(monkeypatch, capfd, cat):
+@pytest.mark.parametrize('test', [
+    # We need lambda here, because pytest will
+    # reuse the parameters, and then the generators
+    # are already used and will not yield anymore
+    ('just text\n', lambda: 'just text'),
+    ('iterable\n', lambda: ["itera", "ble"]),
+    ('abcabc\n', lambda: _test_gen_func),
+    ('abcabc\n', lambda: _test_gen_func()),
+    ('012345\n', lambda: (c for c in range(6))),
+])
+def test_echo_via_pager(monkeypatch, capfd, cat, test):
     monkeypatch.setitem(os.environ, 'PAGER', cat)
     monkeypatch.setattr(click._termui_impl, 'isatty', lambda x: True)
-    click.echo_via_pager('haha')
+
+    expected_output = test[0]
+    test_input = test[1]()
+
+    click.echo_via_pager(test_input)
+
     out, err = capfd.readouterr()
-    assert out == 'haha\n'
+    assert out == expected_output
 
 
 @pytest.mark.skipif(WIN, reason='Test does not make sense on Windows.')