[downloader] repo and downloader improvements

- Repo and Installable
  - converted to abstract classes
    - standard interface lays groundwork for supporting more types
  - tests are more in-depth
  - Repo and subclasses
    - moved to new files
    - postpone reading folder contents until populate() called
    - seperated folder and git repo subtypes (folder and git)
  - no more MISSING_REPO folder, missing is an error now
  - move executor and subprocesses to RepoManager
  - UpdateResult: new object type describing generalized repo updates
- git and pip commands moved to new file and namespace
  - only uses -t {target} if NOT in a venv or virtualenv
- Downloader UI
  - error on invalid repo names
  - cog update command only updates requested cogs if given
    - also shows what cogs were updated, if any
  - better display for list commands

TODO:
- Selective updating (req. by @mikeshardmind)
- Storage of installed cog version
- Use bot_version key in info.json (Cog-Creators#1866)
- Changelogs (epic @ Cog-Creators#1683)

.github/CODEOWNERS

@@ -34,7 +34,7 @@ redbot/cogs/audio/*                 @aikaterna @atiwiex
 redbot/cogs/bank/*                  @tekulvw
 redbot/cogs/cleanup/*               @palmtree5
 redbot/cogs/customcom/*             @palmtree5
-redbot/cogs/downloader/*            @tekulvw
+redbot/cogs/downloader/*            @calebj
 redbot/cogs/economy/*               @palmtree5
 redbot/cogs/filter/*                @palmtree5
 redbot/cogs/general/*               @palmtree5

docs/conf.py

@@ -202,4 +202,5 @@
     "python": ("https://docs.python.org/3.6", None),
     "dpy": ("https://discordpy.readthedocs.io/en/rewrite/", None),
     "motor": ("https://motor.readthedocs.io/en/stable/", None),
+    "packaging": ("https://packaging.pypa.io/en/stable/", None),
 }

docs/framework_downloader.rst

@@ -30,7 +30,7 @@ Keys common to both repo and cog info.json (case sensitive)
 Keys specific to the cog info.json (case sensitive)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- ``bot_version`` (list of integer) - Min version number of Red in the format ``(MAJOR, MINOR, PATCH)``
+- ``bot_version`` - A version string in the format ``"MAJOR.MINOR.PATCH"``
 
 - ``hidden`` (bool) - Determines if a cog is visible in the cog list for a repo.
 
@@ -52,36 +52,66 @@ Keys specific to the cog info.json (case sensitive)
 API Reference
 *************
 
-.. automodule:: redbot.cogs.downloader.json_mixins
+..
+    automodule:: redbot.cogs.downloader.json_mixins
 
-.. autoclass RepoJSONMixin
-    :members
-
-.. automodule:: redbot.cogs.downloader.installable
+..
+    autoclass:: RepoJSONMixin
+    :members:
 
 Installable
 ^^^^^^^^^^^
 
+.. automodule:: redbot.cogs.downloader.installable
+
 .. autoclass:: Installable
     :members:
 
-.. automodule:: redbot.cogs.downloader.repo_manager
+.. autoclass:: FolderInstallable
+    :show-inheritance:
+    :members:
 
 Repo
 ^^^^
 
+.. automodule:: redbot.cogs.downloader.repo
+
 .. autoclass:: Repo
     :members:
 
+.. autoclass:: redbot.cogs.downloader.repos.folder.FolderRepo
+    :show-inheritance:
+    :members:
+
+.. autoclass:: redbot.cogs.downloader.repos.git.GitRepo
+    :show-inheritance:
+    :members:
+
 Repo Manager
 ^^^^^^^^^^^^
 
+.. automodule:: redbot.cogs.downloader.repo_manager
+
 .. autoclass:: RepoManager
     :members:
 
+Update Result
+^^^^^^^^^^^^^
+
+.. automodule:: redbot.cogs.downloader.update_tracker
+
+.. autoclass:: UpdateResult
+
+.. autoclass:: ModuleLists
+
+Utilities
+^^^^^^^^^
+
+.. automodule:: redbot.cogs.downloader.utils
+    :members:
+
 Exceptions
 ^^^^^^^^^^
 
 .. automodule:: redbot.cogs.downloader.errors
     :members:
-

redbot/cogs/downloader/commands.py

@@ -0,0 +1,39 @@
+from types import SimpleNamespace
+
+from .utils import CommandTemplate
+
+COMMANDS = SimpleNamespace(
+    GIT_CHECKOUT_LOCAL=CommandTemplate("git -C {path} checkout {branch} --", extras_kw="paths"),
+    GIT_CHECKOUT_REMOTE=CommandTemplate(
+        "git -C {path} checkout {remote}/{branch} --",
+        extras_kw="paths",
+        defaults={"remote": "origin"},
+    ),
+    GIT_CLONE=CommandTemplate("git clone -b {branch} {url} {folder}"),
+    GIT_CLONE_NO_BRANCH=CommandTemplate("git clone {url} {folder}"),
+    GIT_CURRENT_BRANCH=CommandTemplate("git -C {path} rev-parse --abbrev-ref HEAD"),
+    # GIT_LATEST_COMMIT=CommandTemplate("git -C {path} rev-parse {branch}"),
+    GIT_LATEST_COMMIT=CommandTemplate(
+        "git -C {path} rev-list -1 {branch} --", extras_kw="relative_file_path"
+    ),
+    GIT_HARD_RESET=CommandTemplate(
+        "git -C {path} reset --hard {remote}/{branch} -q", defaults={"remote": "origin"}
+    ),
+    GIT_PULL=CommandTemplate("git -C {path} pull -q --ff-only"),
+    GIT_DIFF_FILE_STATUS=CommandTemplate(
+        "git -C {path} diff --no-commit-id --name-status {old_ref}..{new_ref}"
+    ),
+    GIT_LOG=CommandTemplate(
+        "git -C {path} log --relative-date --reverse {old_ref}.. {relative_file_path}"
+    ),
+    GIT_DISCOVER_REMOTE_URL=CommandTemplate(
+        "git -C {path} config --get remote.{remote}.url", defaults={"remote": "origin"}
+    ),
+    PIP_INSTALL=CommandTemplate("{python} -m pip install -U -t {target_dir}", extras_kw="reqs"),
+    PIP_INSTALL_NO_TARGET=CommandTemplate(
+        "{python} -m pip --disable-pip-version-check install -U", extras_kw="reqs"
+    ),
+    PIP_SHOW=CommandTemplate(
+        "{python} -m pip --disable-pip-version-check show", extras_kw="packages"
+    ),
+)