Move to python git rather than system git (#1296)

* Initial push of moving to use dulwich

* Update help.py

* Fix codestyle

errbot/core_plugins/help.py

@@ -1,7 +1,9 @@
 import textwrap
 import subprocess
 
+from dulwich import errors as dulwich_errors
 from errbot import BotPlugin, botcmd
+from errbot.utils import git_tag_list
 from errbot.version import VERSION
 
 
@@ -12,16 +14,14 @@ class Help(BotPlugin):
 
     def is_git_directory(self, path='.'):
         try:
-            git_call = subprocess.Popen(["git", "tag"], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+            tags = git_tag_list(path)
+        except dulwich_errors.NotGitRepository:
+            tags = None
         except Exception as _:
-            return None
-        tags, _ = git_call.communicate()
-        return_code = git_call.returncode
-        if return_code != 0:
-            return None
-        else:
-            tags = tags.rstrip(b"\n")
-            return tags.split(b"\n").pop(-1)
+            # we might want to handle other exceptions another way. For now leaving this general
+            tags = None
+
+        return tags.pop(-1) if tags is not None else None
 
     # noinspection PyUnusedLocal
     @botcmd(template='about')

errbot/repo_manager.py

@@ -17,6 +17,8 @@
 
 from errbot.storage import StoreMixin
 from errbot.storage.base import StoragePluginBase
+from .utils import git_clone
+from .utils import git_pull
 from .utils import ON_WINDOWS
 
 log = logging.getLogger(__name__)
@@ -244,11 +246,6 @@ def install_repo(self, repo: str) -> str:
         else:
             repo_url = repo
 
-        git_path = which('git')
-        if not git_path:
-            raise RepoException('git command not found: You need to have git installed on '
-                                'your system to be able to install git based plugins.', )
-
         # TODO: Update download path of plugin.
         if repo_url.endswith('tar.gz'):
             fo = urlopen(repo_url)  # nosec
@@ -258,12 +255,10 @@ def install_repo(self, repo: str) -> str:
             human_name = s[:-len('.tar.gz')]
         else:
             human_name = human_name or human_name_for_git_url(repo_url)
-            p = subprocess.Popen([git_path, 'clone', repo_url, human_name], cwd=self.plugin_dir, stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
-            feedback = p.stdout.read().decode('utf-8')
-            error_feedback = p.stderr.read().decode('utf-8')
-            if p.wait():
-                raise RepoException(f'Could not load this plugin: \n\n{feedback}\n\n---\n\n{error_feedback}')
+            try:
+                git_clone(repo_url, os.path.join(self.plugin_dir, human_name))
+            except Exception as exception:  # dulwich errors all base on exceptions.Exception
+                raise RepoException(f'Could not load this plugin: \n\n{repo_url}\n\n---\n\n{exception}')
 
         self.add_plugin_repo(human_name, repo_url)
         return os.path.join(self.plugin_dir, human_name)
@@ -273,24 +268,23 @@ def update_repos(self, repos) -> Generator[Tuple[str, int, str], None, None]:
         This git pulls the specified repos on disk.
         Yields tuples like (name, success, reason)
         """
-        git_path = which('git')
-        if not git_path:
-            yield ('everything', False, 'git command not found: You need to have git installed on '
-                                        'your system to be able to install git based plugins.')
-
         # protects for update outside of what we know is installed
         names = set(self.get_installed_plugin_repos().keys()).intersection(set(repos))
 
         for d in (path.join(self.plugin_dir, name) for name in names):
-            p = subprocess.Popen([git_path, 'pull'], cwd=d, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            feedback = p.stdout.read().decode('utf-8') + '\n' + '-' * 50 + '\n'
-            err = p.stderr.read().strip().decode('utf-8')
-            if err:
-                feedback += err + '\n' + '-' * 50 + '\n'
+            success = 1
+            try:
+                git_pull(d)
+                feedback = "Pulled remote"
+                success = 0
+            except Exception as exception:
+                feedback = f"Error pulling remote {exception}"
+                pass
+
             dep_err, missing_pkgs = check_dependencies(Path(d) / 'requirements.txt')
             if dep_err:
                 feedback += dep_err + '\n'
-            yield d, not p.wait(), feedback
+            yield d, success, feedback
 
     def update_all_repos(self) -> Generator[Tuple[str, int, str], None, None]:
         return self.update_repos(self.get_installed_plugin_repos().keys())

errbot/utils.py

@@ -1,3 +1,5 @@
+from typing import List
+
 import fnmatch
 import inspect
 import logging
@@ -8,6 +10,9 @@
 from platform import system
 from functools import wraps
 
+
+from dulwich import porcelain
+
 log = logging.getLogger(__name__)
 
 ON_WINDOWS = system() == 'Windows'
@@ -188,3 +193,26 @@ def global_restart():
     """Restart the current process."""
     python = sys.executable
     os.execl(python, python, *sys.argv)
+
+
+def git_clone(url: str, path: str) -> None:
+    """
+    Clones a repository from git url to path
+    """
+    if not os.path.exists(path):
+        os.makedirs(path)
+    porcelain.clone(url, path)
+
+
+def git_pull(repo_path: str) -> None:
+    """
+    Does a git pull on a repository
+    """
+    porcelain.pull(repo_path)
+
+
+def git_tag_list(repo_path: str) -> List[str]:
+    """
+    Lists git tags on a cloned repo
+    """
+    porcelain.tag_list(repo_path)

setup.py

@@ -41,6 +41,7 @@
         'ansi',
         'Pygments>=2.0.2',
         'pygments-markdown-lexer>=0.1.0.dev39',  # sytax coloring to debug md
+        'dulwich' # python implementation of git
         ]
 
 if not PY37_OR_GREATER: