Skip to content

Commit

Permalink
Yapsictomy - part 1: botplugins and flows (#1219)
Browse files Browse the repository at this point in the history
* Intermediate state.

* Added a separate plugin info dataclass

* add dataclasses as dep from python < 3.7

* version2array -> version2tuple

* version2tuple -> version2array

* Pass on PluginInfo

* temp

* another pass, running not working

* State where it starts to load plugins

* Cleanup for flows.

* A little bit more tests pass.

* Every tests passes except flows.

* Working on flows

* All tests passes.

* make the linter happy.

* Dataclasses backport doesn't support 3.4+3.5.

* remove 3.4 and 3.5

* error in the travis.yml.
gbin authored and gbin-argo committed May 23, 2018
1 parent 60d8cde commit 9a03b72
Showing 17 changed files with 489 additions and 405 deletions.
4 changes: 0 additions & 4 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -2,10 +2,6 @@ sudo: false
language: python
matrix:
include:
- python: 3.4
env: TOXENV=py34
- python: 3.5
env: TOXENV=py35
- python: 3.6
env: TOXENV=py36
- python: 3.6
9 changes: 4 additions & 5 deletions errbot/core_plugins/backup.py
Original file line number Diff line number Diff line change
@@ -28,13 +28,12 @@ def backup(self, msg, args):

f.write('log.info("Restoring plugins data.")\n')
f.write('bot.plugin_manager.update_dynamic_plugins()\n')
for plug in self._bot.plugin_manager.getAllPlugins():
pobj = plug.plugin_object
if pobj._store:
f.write('pobj = bot.plugin_manager.get_plugin_by_name("' + plug.name + '").plugin_object\n')
for plugin in self._bot.plugin_manager.plugins.values():
if plugin._store:
f.write('pobj = bot.plugin_manager.plugins["' + plugin.name + '"]\n')
f.write('pobj.init_storage()\n')

for key, value in pobj.items():
for key, value in plugin:
f.write('pobj["' + key + '"] = ' + repr(value) + '\n')
f.write('pobj.close_storage()\n')

2 changes: 1 addition & 1 deletion errbot/core_plugins/health.py
Original file line number Diff line number Diff line change
@@ -46,7 +46,7 @@ def status_plugins(self, _, args):
pm = self._bot.plugin_manager
all_blacklisted = pm.get_blacklisted_plugin()
all_loaded = pm.get_all_active_plugin_names()
all_attempted = sorted([p.name for p in pm.all_candidates])
all_attempted = sorted(pm.plugin_infos.keys())
plugins_statuses = []
for name in all_attempted:
if name in all_blacklisted:
2 changes: 1 addition & 1 deletion errbot/core_plugins/plugins.py
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@ def repos_install(self, _, args):
yield 'Some plugins are generating errors:\n' + '\n'.join(errors.values())
# if the load of the plugin failed, uninstall cleanly teh repo
for path in errors.keys():
if path.startswith(local_path):
if str(path).startswith(local_path):
yield 'Removing %s as it did not load correctly.' % local_path
shutil.rmtree(local_path)
else:
6 changes: 3 additions & 3 deletions errbot/core_plugins/vcheck.py
Original file line number Diff line number Diff line change
@@ -5,12 +5,12 @@
import requests

from errbot import BotPlugin
from errbot.utils import version2array
from errbot.utils import version2tuple
from errbot.version import VERSION

HOME = 'http://version.errbot.io/'

installed_version = version2array(VERSION)
installed_version = version2tuple(VERSION)

PY_VERSION = '.'.join(str(e) for e in sys.version_info[:3])

@@ -38,7 +38,7 @@ def _async_vcheck(self):
try:
current_version_txt = requests.get(HOME, params={'errbot': VERSION, 'python': PY_VERSION}).text.strip()
self.log.debug("Tested current Errbot version and it is " + current_version_txt)
current_version = version2array(current_version_txt)
current_version = version2tuple(current_version_txt)
if installed_version < current_version:
self.log.debug('A new version %s has been found, notify the admins !' % current_version)
self.warn_admins(
76 changes: 76 additions & 0 deletions errbot/plugin_info.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
from configparser import ConfigParser
from dataclasses import dataclass
from errbot.utils import version2tuple
from pathlib import Path
from typing import Tuple, List
from configparser import Error as ConfigParserError

VersionType = Tuple[int, int, int]


@dataclass
class PluginInfo:
name: str
module: str
doc: str
core: bool
python_version: VersionType
errbot_minversion: VersionType
errbot_maxversion: VersionType
dependencies: List[str]
location: Path = None

@staticmethod
def load(plugfile_path: Path) -> 'PluginInfo':
with plugfile_path.open(encoding='utf-8') as plugfile:
return PluginInfo.load_file(plugfile, plugfile_path)

@staticmethod
def load_file(plugfile, location: Path) -> 'PluginInfo':
cp = ConfigParser()
cp.read_file(plugfile)
pi = PluginInfo.parse(cp)
pi.location = location
return pi

@staticmethod
def parse(config: ConfigParser) -> 'PluginInfo':
"""
Throws ConfigParserError with a meaningful message if the ConfigParser doesn't contain the minimal
information required.
"""
name = config.get('Core', 'Name')
module = config.get('Core', 'Module')
core = config.get('Core', 'Core', fallback='false').lower() == 'true'
doc = config.get('Documentation', 'Description', fallback=None)

python_version = config.get('Python', 'Version', fallback=None)
# Old format backward compatibility
if python_version:
if python_version in ('2+', '3'):
python_version = (3, 0, 0)
elif python_version == '2':
python_version = (2, 0, 0)
else:
try:
python_version = tuple(version2tuple(python_version)[0:3]) # We can ignore the alpha/beta part.
except ValueError as ve:
raise ConfigParserError('Invalid Python Version format: %s (%s)' % (python_version, ve))

min_version = config.get("Errbot", "Min", fallback=None)
max_version = config.get("Errbot", "Max", fallback=None)
try:
if min_version:
min_version = version2tuple(min_version)
except ValueError as ve:
raise ConfigParserError('Invalid Errbot min version format: %s (%s)' % (min_version, ve))

try:
if max_version:
max_version = version2tuple(max_version)
except ValueError as ve:
raise ConfigParserError('Invalid Errbot max version format: %s (%s)' % (max_version, ve))
depends_on = config.get('Core', 'DependsOn', fallback=None)
deps = [name.strip() for name in depends_on.split(',')] if depends_on else []

return PluginInfo(name, module, doc, core, python_version, min_version, max_version, deps)
622 changes: 281 additions & 341 deletions errbot/plugin_manager.py

Large diffs are not rendered by default.

25 changes: 12 additions & 13 deletions errbot/templating.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
import logging
import os
from errbot.plugin_info import PluginInfo
from jinja2 import Environment, FileSystemLoader
from pathlib import Path

log = logging.getLogger(__name__)


def make_templates_path(root):
return os.path.join(root, 'templates')
def make_templates_path(root: Path) -> Path:
return root / 'templates'


system_templates_path = make_templates_path(os.path.dirname(__file__))
system_templates_path = str(make_templates_path(Path(__file__).parent))
template_path = [system_templates_path]
env = Environment(loader=FileSystemLoader(template_path),
trim_blocks=True,
@@ -21,25 +23,22 @@ def tenv():
return env


def make_templates_from_plugin_path(plugin_path):
return make_templates_path(os.path.dirname(plugin_path))


def add_plugin_templates_path(path):
def add_plugin_templates_path(plugin_info: PluginInfo):
global env
tmpl_path = make_templates_from_plugin_path(path)
if os.path.exists(tmpl_path):
tmpl_path = make_templates_path(plugin_info.location.parent)
if tmpl_path.exists():
log.debug("Templates directory found for this plugin [%s]" % tmpl_path)
template_path.append(tmpl_path)
template_path.append(str(tmpl_path)) # for webhooks

# Ditch and recreate a new templating environment
env = Environment(loader=FileSystemLoader(template_path), autoescape=True)
return
log.debug("No templates directory found for this plugin [Looking for %s]" % tmpl_path)


def remove_plugin_templates_path(path):
def remove_plugin_templates_path(plugin_info: PluginInfo):
global env
tmpl_path = make_templates_from_plugin_path(path)
tmpl_path = str(make_templates_path(plugin_info.location.parent))
if tmpl_path in template_path:
template_path.remove(tmpl_path)
# Ditch and recreate a new templating environment
4 changes: 2 additions & 2 deletions errbot/utils.py
Original file line number Diff line number Diff line change
@@ -77,7 +77,7 @@ def get_class_for_method(meth):
INVALID_VERSION_EXCEPTION = 'version %s in not in format "x.y.z" or "x.y.z-{beta,alpha,rc1,rc2...}" for example "1.2.2"'


def version2array(version):
def version2tuple(version):
vsplit = version.split('-')

if len(vsplit) == 2:
@@ -103,7 +103,7 @@ def version2array(version):
if len(response) != 4:
raise ValueError(INVALID_VERSION_EXCEPTION % version)

return response
return tuple(response)


def unescape_xml(text):
3 changes: 3 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@

py_version = sys.version_info[:2]
PY35_OR_GREATER = py_version >= (3, 5)
PY37_OR_GREATER = py_version >= (3, 7)

ON_WINDOWS = system() == 'Windows'

@@ -48,6 +49,8 @@
if not PY35_OR_GREATER:
deps += ['typing', ] # backward compatibility for 3.3 and 3.4

if not PY37_OR_GREATER:
deps += ['dataclasses'] # backward compatibility for 3.3->3.6 for dataclasses

if not ON_WINDOWS:
deps += ['daemonize']
3 changes: 2 additions & 1 deletion tests/base_backend_test.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# coding=utf-8
import sys
import logging
from pathlib import Path
from tempfile import mkdtemp
from os.path import sep

@@ -276,7 +277,7 @@ def dummy_execute_and_send():
example_message.to = dummy.build_identifier('err')

assets_path = os.path.join(os.path.dirname(__file__), 'assets')
templating.template_path.append(templating.make_templates_path(assets_path))
templating.template_path.append(str(templating.make_templates_path(Path(assets_path))))
templating.env = templating.Environment(loader=templating.FileSystemLoader(templating.template_path))
return dummy, example_message

3 changes: 1 addition & 2 deletions tests/commands_test.py
Original file line number Diff line number Diff line change
@@ -153,8 +153,7 @@ def test_broken_plugin(testbot):
tgz = os.path.join(tempd, "borken.tar.gz")
with tarfile.open(tgz, "w:gz") as tar:
tar.add(borken_plugin_dir, arcname='borken')
assert 'Installing' in testbot.exec_command('!repos install file://' + tgz,
timeout=120)
assert 'Installing' in testbot.exec_command('!repos install file://' + tgz, timeout=120)
assert 'import borken # fails' in testbot.pop_message()
assert 'as it did not load correctly.' in testbot.pop_message()
assert 'Plugins reloaded.' in testbot.pop_message()
8 changes: 4 additions & 4 deletions tests/dependencies_test.py
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@ def test_if_all_loaded_by_default(testbot):
def test_single_dependency(testbot):
pm = testbot.bot.plugin_manager
for p in ('Single', 'Parent1', 'Parent2'):
pm.deactivate_plugin_by_name(p)
pm.deactivate_plugin(p)

# everything should be gone
plug_names = pm.get_all_active_plugin_names()
@@ -34,7 +34,7 @@ def test_double_dependency(testbot):
pm = testbot.bot.plugin_manager
all = ('Double', 'Parent1', 'Parent2')
for p in all:
pm.deactivate_plugin_by_name(p)
pm.deactivate_plugin(p)

pm.activate_plugin('Double')
plug_names = pm.get_all_active_plugin_names()
@@ -46,12 +46,12 @@ def test_dependency_retrieval(testbot):
assert 'youpi' in testbot.exec_command('!depfunc')


def test_direct_cicular_dependency(testbot):
def test_direct_circular_dependency(testbot):
plug_names = testbot.bot.plugin_manager.get_all_active_plugin_names()
assert 'Circular1' not in plug_names


def test_indirect_cicular_dependency(testbot):
def test_indirect_circular_dependency(testbot):
plug_names = testbot.bot.plugin_manager.get_all_active_plugin_names()
assert 'Circular2' not in plug_names
assert 'Circular3' not in plug_names
66 changes: 66 additions & 0 deletions tests/plugin_info_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import sys

import pytest
from io import StringIO
from pathlib import Path
from errbot.plugin_info import PluginInfo

plugfile_base = Path(__file__).absolute().parent / 'config_plugin'
plugfile_path = plugfile_base / 'config.plug'


def test_load_from_plugfile_path():
pi = PluginInfo.load(plugfile_path)
assert pi.name == 'Config'
assert pi.module == 'config'
assert pi.doc is None
assert pi.python_version == (3, 0, 0)
assert pi.errbot_minversion is None
assert pi.errbot_maxversion is None


@pytest.mark.parametrize('test_input,expected', [
('2', (2, 0, 0)),
('2+', (3, 0, 0)),
('3', (3, 0, 0)),
('1.2.3', (1, 2, 3)),
('1.2.3-beta', (1, 2, 3)),
])
def test_python_version_parse(test_input, expected):
f = StringIO("""
[Core]
Name = Config
Module = config
[Python]
Version = %s
""" % test_input)

assert PluginInfo.load_file(f, None).python_version == expected


def test_doc():
f = StringIO("""
[Core]
Name = Config
Module = config
[Documentation]
Description = something
""")

assert PluginInfo.load_file(f, None).doc == 'something'


def test_errbot_version():
f = StringIO("""
[Core]
Name = Config
Module = config
[Errbot]
Min = 1.2.3
Max = 4.5.6-beta
""")
info = PluginInfo.load_file(f, None)
assert info.errbot_minversion == (1, 2, 3, sys.maxsize)
assert info.errbot_maxversion == (4, 5, 6, 0)
55 changes: 30 additions & 25 deletions tests/plugin_management_test.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
import os
import pytest
import tempfile
from configparser import ConfigParser

from errbot import plugin_manager
from errbot.plugin_info import PluginInfo
from errbot.plugin_manager import IncompatiblePluginException
from errbot.utils import find_roots, collect_roots

CORE_PLUGINS = plugin_manager.CORE_PLUGINS
@@ -80,54 +83,56 @@ def test_ignore_dotted_directories():
assert collect_roots((CORE_PLUGINS, root)) == {CORE_PLUGINS, }


def dummy_config_parser() -> ConfigParser:
cp = ConfigParser()
cp.add_section('Core')
cp.set('Core', 'Name', 'dummy')
cp.set('Core', 'Module', 'dummy')
cp.add_section('Errbot')
return cp


def test_errbot_version_check():
real_version = plugin_manager.VERSION

too_high_min_1 = ConfigParser()
too_high_min_1.add_section('Errbot')
too_high_min_1 = dummy_config_parser()
too_high_min_1.set('Errbot', 'Min', '1.6.0')

too_high_min_2 = ConfigParser()
too_high_min_2.add_section('Errbot')
too_high_min_2 = dummy_config_parser()
too_high_min_2.set('Errbot', 'Min', '1.6.0')
too_high_min_2.set('Errbot', 'Max', '2.0.0')

too_low_max_1 = ConfigParser()
too_low_max_1.add_section('Errbot')
too_low_max_1 = dummy_config_parser()
too_low_max_1.set('Errbot', 'Max', '1.0.1-beta')

too_low_max_2 = ConfigParser()
too_low_max_2.add_section('Errbot')
too_low_max_2 = dummy_config_parser()
too_low_max_2.set('Errbot', 'Min', '0.9.0-rc2')
too_low_max_2.set('Errbot', 'Max', '1.0.1-beta')

ok1 = ConfigParser() # no section

ok2 = ConfigParser()
ok2.add_section('Errbot') # empty section
ok1 = dummy_config_parser() # empty section

ok3 = ConfigParser()
ok3.add_section('Errbot')
ok3.set('Errbot', 'Min', '1.4.0')
ok2 = dummy_config_parser()
ok2.set('Errbot', 'Min', '1.4.0')

ok4 = ConfigParser()
ok4.add_section('Errbot')
ok4.set('Errbot', 'Max', '1.5.2')
ok3 = dummy_config_parser()
ok3.set('Errbot', 'Max', '1.5.2')

ok5 = ConfigParser()
ok5.add_section('Errbot')
ok5 .set('Errbot', 'Min', '1.2.1')
ok5 .set('Errbot', 'Max', '1.6.1-rc1')
ok4 = dummy_config_parser()
ok4.set('Errbot', 'Min', '1.2.1')
ok4.set('Errbot', 'Max', '1.6.1-rc1')

try:
plugin_manager.VERSION = '1.5.2'
for config in (too_high_min_1,
too_high_min_2,
too_low_max_1,
too_low_max_2):
assert not plugin_manager.check_errbot_plug_section('should_fail', config)
pi = PluginInfo.parse(config)
with pytest.raises(IncompatiblePluginException):
plugin_manager.check_errbot_version(pi)

for config in (ok1, ok2, ok3, ok4, ok5):
assert plugin_manager.check_errbot_plug_section('should_work', config)
for config in (ok1, ok2, ok3, ok4):
pi = PluginInfo.parse(config)
plugin_manager.check_errbot_version(pi)
finally:
plugin_manager.VERSION = real_version
4 changes: 2 additions & 2 deletions tests/utils_test.py
Original file line number Diff line number Diff line change
@@ -24,7 +24,7 @@
('2.0.0-beta', '2.0.1'),
])
def test_version_check(v1, v2):
assert version2array(v1) < version2array(v2)
assert version2tuple(v1) < version2tuple(v2)


@pytest.mark.parametrize('version', [
@@ -36,7 +36,7 @@ def test_version_check(v1, v2):
])
def test_version_check_negative(version):
with pytest.raises(ValueError):
version2array(version)
version2tuple(version)


def test_formattimedelta():
2 changes: 1 addition & 1 deletion tox.ini
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[tox]
envlist = py34,py35,py36,codestyle,pypi-lint
envlist = py36,codestyle,pypi-lint
skip_missing_interpreters = True

[testenv]

0 comments on commit 9a03b72

Please sign in to comment.