From b640cbd8d18a77a6fc1b356dc4b06300ad2dd3b0 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 21 Jul 2019 13:04:19 +0100 Subject: [PATCH 001/167] :sparkles: initial support of pythonfilesystem2 in data loader --- .moban.cd/moban.yml | 1 + min_requirements.txt | 1 + moban/data_loaders/json_loader.py | 10 +++++++--- moban/data_loaders/manager.py | 4 ++-- moban/data_loaders/yaml.py | 13 +++++++++---- requirements.txt | 1 + setup.py | 1 + 7 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 2a3dca1f..fed0c7f6 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -28,6 +28,7 @@ dependencies: - crayons>= 0.1.0 - GitPython>=2.0.0 - git-url-parse>=1.2.2 + - fs description: Yet another jinja2 cli command for static text generation scm_host: github.com lint_command: make install_test format git-diff-check lint diff --git a/min_requirements.txt b/min_requirements.txt index b5c854ba..fe5d153d 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -8,3 +8,4 @@ appdirs==1.2.0 crayons== 0.1.0 GitPython==2.0.0 git-url-parse==1.2.2 +fs diff --git a/moban/data_loaders/json_loader.py b/moban/data_loaders/json_loader.py index b5726fee..34ceadd2 100644 --- a/moban/data_loaders/json_loader.py +++ b/moban/data_loaders/json_loader.py @@ -1,3 +1,4 @@ +from fs import path, open_fs import json from moban import constants @@ -9,6 +10,9 @@ def open_json(file_name): """ returns json contents as string """ - with open(file_name, "r") as json_data: - data = json.load(json_data) - return data + dir_name = path.dirname(file_name) + the_file_name = path.basename(file_name) + with open_fs(dir_name) as the_fs: + with the_fs.open(the_file_name) as json_file: + data = json.load(json_file) + return data diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index 3dd402a9..a62cf89e 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -1,4 +1,4 @@ -import os +from fs import path from moban import utils, constants from lml.plugin import PluginManager @@ -9,7 +9,7 @@ def __init__(self): super(AnyDataLoader, self).__init__(constants.DATA_LOADER_EXTENSION) def get_data(self, file_name): - file_extension = os.path.splitext(file_name)[1] + file_extension = path.splitext(file_name)[1] file_type = file_extension if file_extension.startswith("."): file_type = file_type[1:] diff --git a/moban/data_loaders/yaml.py b/moban/data_loaders/yaml.py index 8fce4710..dbf8b756 100644 --- a/moban/data_loaders/yaml.py +++ b/moban/data_loaders/yaml.py @@ -1,3 +1,5 @@ +from fs import path, open_fs + from moban import constants from lml.plugin import PluginInfo from ruamel.yaml import YAML @@ -5,7 +7,10 @@ @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["yaml", "yml"]) def open_yaml(file_name): - with open(file_name, "r") as data_yaml: - yaml = YAML(typ="rt") - data = yaml.load(data_yaml) - return data + dir_name = path.dirname(file_name) + the_file_name = path.basename(file_name) + with open_fs(dir_name) as the_fs: + with the_fs.open(the_file_name) as data_yaml: + yaml = YAML(typ="rt") + data = yaml.load(data_yaml) + return data diff --git a/requirements.txt b/requirements.txt index 2305c66a..481bf3da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ appdirs>=1.2.0 crayons>= 0.1.0 GitPython>=2.0.0 git-url-parse>=1.2.2 +fs diff --git a/setup.py b/setup.py index a7564107..5dbdc995 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ "crayons>= 0.1.0", "GitPython>=2.0.0", "git-url-parse>=1.2.2", + "fs", ] SETUP_COMMANDS = {} From 1889142267f3464890e54bd9a1bac11ca00f98c4 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 21 Jul 2019 13:10:24 +0100 Subject: [PATCH 002/167] :sparkles: use pythonfilesystem2 to write bytes out --- moban/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 87e0f63b..cf15ef65 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -5,6 +5,7 @@ import logging from moban import constants, exceptions +from fs import path as fs_path, open_fs log = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -66,8 +67,11 @@ def write_file_out(filename, content): dest_folder = os.path.dirname(filename) if dest_folder: mkdir_p(dest_folder) - with open(filename, "wb") as out: - out.write(content) + + dir_name = fs_path.dirname(filename) + the_file_name = fs_path.basename(filename) + with open_fs(dir_name) as the_fs: + the_fs.writebytes(the_file_name, content) def mkdir_p(path): From 3ec168abd45d49d8acaf6ce1b3b2d69b07021075 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 22 Jul 2019 22:55:34 +0100 Subject: [PATCH 003/167] :bug: convert ascii to unicode in python 2 --- moban/data_loaders/json_loader.py | 5 +++++ moban/data_loaders/yaml.py | 5 +++++ moban/utils.py | 2 ++ 3 files changed, 12 insertions(+) diff --git a/moban/data_loaders/json_loader.py b/moban/data_loaders/json_loader.py index 34ceadd2..5d8187b4 100644 --- a/moban/data_loaders/json_loader.py +++ b/moban/data_loaders/json_loader.py @@ -1,15 +1,20 @@ +import sys from fs import path, open_fs import json from moban import constants from lml.plugin import PluginInfo +PY2 = sys.version_info[0] == 2 + @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["json"]) def open_json(file_name): """ returns json contents as string """ + if PY2: + file_name = unicode(file_name) dir_name = path.dirname(file_name) the_file_name = path.basename(file_name) with open_fs(dir_name) as the_fs: diff --git a/moban/data_loaders/yaml.py b/moban/data_loaders/yaml.py index dbf8b756..3605f5d1 100644 --- a/moban/data_loaders/yaml.py +++ b/moban/data_loaders/yaml.py @@ -1,12 +1,17 @@ +import sys from fs import path, open_fs from moban import constants from lml.plugin import PluginInfo from ruamel.yaml import YAML +PY2 = sys.version_info[0] == 2 + @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["yaml", "yml"]) def open_yaml(file_name): + if PY2: + file_name = unicode(file_name) dir_name = path.dirname(file_name) the_file_name = path.basename(file_name) with open_fs(dir_name) as the_fs: diff --git a/moban/utils.py b/moban/utils.py index cf15ef65..a9c17a12 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -68,6 +68,8 @@ def write_file_out(filename, content): if dest_folder: mkdir_p(dest_folder) + if PY2: + filename = unicode(filename) dir_name = fs_path.dirname(filename) the_file_name = fs_path.basename(filename) with open_fs(dir_name) as the_fs: From d800ef869841d56a4b5f9ab4bdeeb6f9fd552c46 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 22 Jul 2019 23:08:44 +0100 Subject: [PATCH 004/167] :green_heart: fix linting issue --- moban/data_loaders/json_loader.py | 6 ++++-- moban/data_loaders/manager.py | 4 ++-- moban/data_loaders/yaml.py | 6 ++++-- moban/hashstore.py | 5 +++-- moban/utils.py | 12 ++++++++---- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/moban/data_loaders/json_loader.py b/moban/data_loaders/json_loader.py index 5d8187b4..84cdadb9 100644 --- a/moban/data_loaders/json_loader.py +++ b/moban/data_loaders/json_loader.py @@ -1,10 +1,11 @@ import sys -from fs import path, open_fs import json from moban import constants from lml.plugin import PluginInfo +from fs import path, open_fs + PY2 = sys.version_info[0] == 2 @@ -14,7 +15,8 @@ def open_json(file_name): returns json contents as string """ if PY2: - file_name = unicode(file_name) + if isinstance(file_name, unicode) is False: + file_name = unicode(file_name) dir_name = path.dirname(file_name) the_file_name = path.basename(file_name) with open_fs(dir_name) as the_fs: diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index a62cf89e..86dcd16e 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -1,8 +1,8 @@ -from fs import path - from moban import utils, constants from lml.plugin import PluginManager +from fs import path + class AnyDataLoader(PluginManager): def __init__(self): diff --git a/moban/data_loaders/yaml.py b/moban/data_loaders/yaml.py index 3605f5d1..f5ed4bf1 100644 --- a/moban/data_loaders/yaml.py +++ b/moban/data_loaders/yaml.py @@ -1,17 +1,19 @@ import sys -from fs import path, open_fs from moban import constants from lml.plugin import PluginInfo from ruamel.yaml import YAML +from fs import path, open_fs + PY2 = sys.version_info[0] == 2 @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["yaml", "yml"]) def open_yaml(file_name): if PY2: - file_name = unicode(file_name) + if isinstance(file_name, unicode) is False: + file_name = unicode(file_name) dir_name = path.dirname(file_name) the_file_name = path.basename(file_name) with open_fs(dir_name) as the_fs: diff --git a/moban/hashstore.py b/moban/hashstore.py index 25b72518..8e2b07d5 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -65,8 +65,9 @@ def get_file_hash(afile): def get_hash(content): md5 = hashlib.md5() - if PY2 and content.__class__.__name__ == "unicode": - content = content.encode("utf-8") + if PY2: + if isinstance(content, unicode): + content = content.encode("utf-8") md5.update(content) return md5.digest().decode("latin1") diff --git a/moban/utils.py b/moban/utils.py index a9c17a12..6b9fe4b0 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -5,7 +5,9 @@ import logging from moban import constants, exceptions -from fs import path as fs_path, open_fs + +from fs import path as fs_path +from fs import open_fs log = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -62,14 +64,16 @@ def file_permissions(afile): def write_file_out(filename, content): - if PY2 and content.__class__.__name__ == "unicode": - content = content.encode("utf-8") + if PY2: + if isinstance(content, unicode): + content = content.encode("utf-8") dest_folder = os.path.dirname(filename) if dest_folder: mkdir_p(dest_folder) if PY2: - filename = unicode(filename) + if isinstance(filename, unicode): + filename = unicode(filename) dir_name = fs_path.dirname(filename) the_file_name = fs_path.basename(filename) with open_fs(dir_name) as the_fs: From 9249ed3dc44a30f7dc205b5c8d2683613f2e9b76 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 22 Jul 2019 23:14:07 +0100 Subject: [PATCH 005/167] :green_heart: fix file writing --- moban/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/utils.py b/moban/utils.py index 6b9fe4b0..1cb1233c 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -72,7 +72,7 @@ def write_file_out(filename, content): mkdir_p(dest_folder) if PY2: - if isinstance(filename, unicode): + if isinstance(filename, unicode) is False: filename = unicode(filename) dir_name = fs_path.dirname(filename) the_file_name = fs_path.basename(filename) From 03c3d4b48ec7ac3adbea5a63de857d371d5cf78c Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 22 Jul 2019 23:27:42 +0100 Subject: [PATCH 006/167] :fire: ignore unicode warning --- lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint.sh b/lint.sh index 08f7c655..a0071dff 100644 --- a/lint.sh +++ b/lint.sh @@ -1 +1 @@ -flake8 --max-line-length=88 --exclude=.moban.d,docs --ignore=W503,W504 +flake8 --max-line-length=88 --exclude=.moban.d,docs --ignore=W503,W504 --builtins=unicode From 97553157caec28573ccdd0a2a1c88d7c53dd4f4b Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 08:05:32 +0100 Subject: [PATCH 007/167] :microscope: update context with fs.path.join --- tests/test_context.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_context.py b/tests/test_context.py index 23b64133..b1f3187e 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,11 +1,12 @@ import os +import fs.path from nose.tools import eq_ from moban.plugins.context import Context def test_context(): - context = Context(os.path.join("tests", "fixtures")) + context = Context(fs.path.join("tests", "fixtures")) data = context.get_data("simple.yaml") eq_(data["simple"], "yaml") @@ -14,7 +15,7 @@ def test_environ_variables(): test_var = "TEST_ENVIRONMENT_VARIABLE" test_value = "am I found" os.environ[test_var] = test_value - context = Context(os.path.join("tests", "fixtures")) + context = Context(fs.path.join("tests", "fixtures")) data = context.get_data("simple.yaml") eq_(data[test_var], test_value) @@ -23,7 +24,7 @@ def test_json_data_overrides_environ_variables(): test_var = "TEST_ENVIRONMENT_VARIABLE" test_value = "am I found" os.environ[test_var] = test_value - context = Context(os.path.join("tests", "fixtures")) + context = Context(fs.path.join("tests", "fixtures")) data = context.get_data("simple.json") eq_(data[test_var], test_value) @@ -32,6 +33,6 @@ def test_unknown_data_file(): test_var = "TEST_ENVIRONMENT_VARIABLE" test_value = "am I found" os.environ[test_var] = test_value - context = Context(os.path.join("tests", "fixtures")) + context = Context(fs.path.join("tests", "fixtures")) data = context.get_data("unknown.data") eq_(data[test_var], test_value) From daf6956354cfd37fe4d084e8266655551a991a90 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 08:51:19 +0100 Subject: [PATCH 008/167] :green_heart: updated all os.path with fs.path --- moban/mobanfile/templates.py | 87 ++++++++++++++++++++--------- moban/utils.py | 7 ++- tests/fixtures/duplicated.moban.yml | 4 +- tests/mobanfile/test_templates.py | 3 +- 4 files changed, 68 insertions(+), 33 deletions(-) diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 795b15cd..611482f0 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -1,10 +1,34 @@ import os +import sys +import fs +import fs.path import logging from moban import reporter from moban.utils import find_file_in_template_dirs log = logging.getLogger(__name__) +PY2 = sys.version_info[0] == 2 + + +def is_dir(path): + if PY2: + if isinstance(path, unicode) is False: + path = unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as the_fs: + return the_fs.isdir(the_file_name) + + +def is_file(path): + if PY2: + if isinstance(path, unicode) is False: + path = unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as the_fs: + return the_fs.isfile(the_file_name) def handle_template(template_file, output, template_dirs): @@ -29,7 +53,7 @@ def handle_template(template_file, output, template_dirs): reporter.report_error_message( "{0} cannot be found".format(template_file) ) - elif os.path.isdir(template_file_on_disk): + elif is_dir(template_file_on_disk): for a_triple in _list_dir_files( template_file, template_file_on_disk, output ): @@ -40,37 +64,48 @@ def handle_template(template_file, output, template_dirs): def _list_dir_files(source, actual_source_path, dest): - for file_name in os.listdir(actual_source_path): - real_src_file = os.path.join(actual_source_path, file_name) - if os.path.isfile(real_src_file): - # please note jinja2 does NOT like windows path - # hence the following statement looks like cross platform - # src_file_under_dir = os.path.join(source, file_name) - # but actually it breaks windows instead. - src_file_under_dir = "%s/%s" % (source, file_name) - - dest_file_under_dir = os.path.join(dest, file_name) - template_type = _get_template_type(src_file_under_dir) - yield (src_file_under_dir, dest_file_under_dir, template_type) + if PY2: + if isinstance(path, unicode) is False: + actual_source_path = unicode(actual_source_path) + dir_name = fs.path.dirname(actual_source_path) + the_file_name = fs.path.basename(actual_source_path) + with fs.open_fs(dir_name) as fs_system: + for file_name in fs_system.listdir(the_file_name): + if fs_system.isfile(fs.path.join(the_file_name, file_name)): + # please note jinja2 does NOT like windows path + # hence the following statement looks like cross platform + # src_file_under_dir = os.path.join(source, file_name) + # but actually it breaks windows instead. + src_file_under_dir = "%s/%s" % (source, file_name) + + dest_file_under_dir = fs.path.join(dest, file_name) + template_type = _get_template_type(src_file_under_dir) + yield (src_file_under_dir, dest_file_under_dir, template_type) def _listing_directory_files_recusively(source, actual_source_path, dest): - for file_name in os.listdir(actual_source_path): - src_file_under_dir = os.path.join(source, file_name) - dest_file_under_dir = os.path.join(dest, file_name) - real_src_file = os.path.join(actual_source_path, file_name) - if os.path.isfile(real_src_file): - template_type = _get_template_type(src_file_under_dir) - yield (src_file_under_dir, dest_file_under_dir, template_type) - elif os.path.isdir(real_src_file): - for a_triple in _listing_directory_files_recusively( - src_file_under_dir, real_src_file, dest_file_under_dir - ): - yield a_triple + if PY2: + if isinstance(path, unicode) is False: + actual_source_path = unicode(actual_source_path) + dir_name = fs.path.dirname(actual_source_path) + the_file_name = fs.path.basename(actual_source_path) + with fs.open_fs(dir_name) as fs_system: + for file_name in fs_system.listdir(the_file_name): + src_file_under_dir = fs.path.join(source, file_name) + dest_file_under_dir = fs.path.join(dest, file_name) + real_src_file = fs.path.join(actual_source_path, file_name) + if fs_system.isfile(fs.path.join(the_file_name, file_name)): + template_type = _get_template_type(src_file_under_dir) + yield (src_file_under_dir, dest_file_under_dir, template_type) + elif fs_system.isdir(fs.path.join(the_file_name, file_name)): + for a_triple in _listing_directory_files_recusively( + src_file_under_dir, real_src_file, dest_file_under_dir + ): + yield a_triple def _get_template_type(template_file): - _, extension = os.path.splitext(template_file) + _, extension = fs.path.splitext(template_file) if extension: template_type = extension[1:] else: diff --git a/moban/utils.py b/moban/utils.py index 1cb1233c..9fab85f2 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,4 +1,5 @@ import os +import fs import sys import stat import errno @@ -138,8 +139,8 @@ def verify_the_existence_of_directories(dirs): def find_file_in_template_dirs(src, template_dirs): log.debug(template_dirs) for folder in template_dirs: - path = os.path.join(folder, src) - if os.path.exists(path): - return path + with fs.open_fs(folder) as fs_system: + if fs_system.exists(src): + return fs_system.getsyspath(src) else: return None diff --git a/tests/fixtures/duplicated.moban.yml b/tests/fixtures/duplicated.moban.yml index 2d4407b1..e2b4c0c6 100644 --- a/tests/fixtures/duplicated.moban.yml +++ b/tests/fixtures/duplicated.moban.yml @@ -1,8 +1,6 @@ configuration: - configuration_dir: "commons/config" template_dir: - - "commons/templates" - - ".moban.d" + - "." targets: - setup.py: setup.py - setup.py: setup.py diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index 21d90f2a..ae37fd02 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -1,4 +1,5 @@ import os +import fs.path from mock import patch from nose.tools import eq_ @@ -7,7 +8,7 @@ class TestHandleTemplateFunction: def setUp(self): - self.base_dir = [os.path.join("tests", "fixtures")] + self.base_dir = [fs.path.join("tests", "fixtures")] def test_copy_files(self): results = list( From e1322e46cc168bffe6e5614413003e820ed1383c Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 08:51:38 +0100 Subject: [PATCH 009/167] :lipstick: update coding style --- moban/mobanfile/templates.py | 7 ++++--- moban/utils.py | 2 +- tests/mobanfile/test_templates.py | 3 ++- tests/test_context.py | 3 ++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 611482f0..27b38648 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -1,12 +1,13 @@ import os import sys -import fs -import fs.path import logging from moban import reporter from moban.utils import find_file_in_template_dirs +import fs +import fs.path + log = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -77,7 +78,7 @@ def _list_dir_files(source, actual_source_path, dest): # src_file_under_dir = os.path.join(source, file_name) # but actually it breaks windows instead. src_file_under_dir = "%s/%s" % (source, file_name) - + dest_file_under_dir = fs.path.join(dest, file_name) template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) diff --git a/moban/utils.py b/moban/utils.py index 9fab85f2..d8435ee3 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,5 +1,4 @@ import os -import fs import sys import stat import errno @@ -7,6 +6,7 @@ from moban import constants, exceptions +import fs from fs import path as fs_path from fs import open_fs diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index ae37fd02..b8cf1e8c 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -1,10 +1,11 @@ import os -import fs.path from mock import patch from nose.tools import eq_ from moban.mobanfile.templates import handle_template +import fs.path + class TestHandleTemplateFunction: def setUp(self): diff --git a/tests/test_context.py b/tests/test_context.py index b1f3187e..e74019c9 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,9 +1,10 @@ import os -import fs.path from nose.tools import eq_ from moban.plugins.context import Context +import fs.path + def test_context(): context = Context(fs.path.join("tests", "fixtures")) From db5f90a6ad1d3ec313773f46b4e217b561c4e832 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 08:57:07 +0100 Subject: [PATCH 010/167] :hammer: more replacement in test targets --- tests/mobanfile/test_targets.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/mobanfile/test_targets.py b/tests/mobanfile/test_targets.py index 283ecf2c..18b7b93a 100644 --- a/tests/mobanfile/test_targets.py +++ b/tests/mobanfile/test_targets.py @@ -1,4 +1,3 @@ -import os import uuid from nose.tools import eq_, raises @@ -6,10 +5,12 @@ from moban.exceptions import GroupTargetNotFound from moban.definitions import TemplateTarget +import fs.path + TEMPLATE = "a.jj2" OUTPUT = "a.output" CONFIGURATION = "data.config" -TEMPLATE_DIRS = [os.path.join("tests", "fixtures")] +TEMPLATE_DIRS = [fs.path.join("tests", "fixtures")] DEFAULT_TEMPLATE_TYPE = "default-template-type" From d723fe1744e792c07c5c0ca303fd191c1795396d Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 09:02:30 +0100 Subject: [PATCH 011/167] :tractor: further curb of os.path --- tests/test_regression.py | 8 +++++--- tests/test_template.py | 20 +++++++++++--------- tests/test_yaml_loader.py | 20 ++++++++++---------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/tests/test_regression.py b/tests/test_regression.py index f80e8336..894267e7 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -7,6 +7,8 @@ from moban.main import main from nose.tools import eq_ +import fs.path + def custom_dedent(long_texts): refined = dedent(long_texts) @@ -25,13 +27,13 @@ def test_coping_binary_file(self): self._raw_moban( args, folder, - os.path.join("copy-source", "image.png"), + fs.path.join("copy-source", "image.png"), "regression-test.png", ) def _raw_moban(self, args, folder, expected, output): - base_dir = os.path.join("tests", "regression_tests") - os.chdir(os.path.join(base_dir, folder)) + base_dir = fs.path.join("tests", "regression_tests") + os.chdir(fs.path.join(base_dir, folder)) with patch.object(sys, "argv", args): main() status = filecmp.cmp(output, expected) diff --git a/tests/test_template.py b/tests/test_template.py index 4c495195..f4184432 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -7,6 +7,8 @@ from moban.jinja2.engine import Engine from moban.data_loaders.yaml import open_yaml +import fs.path + MODULE = "moban.plugins.template" @@ -57,12 +59,12 @@ def test_do_templates_2(_do_templates_with_more_shared_templates): def test_do_templates_with_more_shared_templates(): - base_dir = os.path.join("tests", "fixtures") + base_dir = fs.path.join("tests", "fixtures") engine = ENGINES.get_engine( - "jinja2", base_dir, os.path.join(base_dir, "config") + "jinja2", base_dir, fs.path.join(base_dir, "config") ) engine._render_with_finding_template_first( - {"a.jj2": [(os.path.join(base_dir, "child.yaml"), "test")]} + {"a.jj2": [(fs.path.join(base_dir, "child.yaml"), "test")]} ) with open("test", "r") as f: content = f.read() @@ -71,12 +73,12 @@ def test_do_templates_with_more_shared_templates(): def test_do_templates_with_more_shared_data(): - base_dir = os.path.join("tests", "fixtures") + base_dir = fs.path.join("tests", "fixtures") engine = ENGINES.get_engine( - "jinja2", base_dir, os.path.join(base_dir, "config") + "jinja2", base_dir, fs.path.join(base_dir, "config") ) engine._render_with_finding_data_first( - {os.path.join(base_dir, "child.yaml"): [("a.jj2", "test")]} + {fs.path.join(base_dir, "child.yaml"): [("a.jj2", "test")]} ) with open("test", "r") as f: content = f.read() @@ -85,7 +87,7 @@ def test_do_templates_with_more_shared_data(): def test_get_user_defined_engine(): - test_fixture = os.path.join( + test_fixture = fs.path.join( "tests", "fixtures", "mobanengine", "sample_template_type.yml" ) template_types = open_yaml(test_fixture) @@ -95,7 +97,7 @@ def test_get_user_defined_engine(): def test_custom_file_extension_is_assocated_with_user_defined_engine(): - test_fixture = os.path.join( + test_fixture = fs.path.join( "tests", "fixtures", "mobanengine", "sample_template_type.yml" ) template_types = open_yaml(test_fixture) @@ -105,7 +107,7 @@ def test_custom_file_extension_is_assocated_with_user_defined_engine(): def test_built_in_jinja2_file_extension_still_works(): - test_fixture = os.path.join( + test_fixture = fs.path.join( "tests", "fixtures", "mobanengine", "sample_template_type.yml" ) template_types = open_yaml(test_fixture) diff --git a/tests/test_yaml_loader.py b/tests/test_yaml_loader.py index fe3cefe5..ce453c8b 100644 --- a/tests/test_yaml_loader.py +++ b/tests/test_yaml_loader.py @@ -1,35 +1,35 @@ -import os - from nose.tools import eq_, raises from moban.data_loaders.yaml import open_yaml from moban.data_loaders.manager import load_data +import fs.path + def test_simple_yaml(): - test_file = os.path.join("tests", "fixtures", "simple.yaml") + test_file = fs.path.join("tests", "fixtures", "simple.yaml") data = open_yaml(test_file) eq_(data, {"simple": "yaml"}) def test_inheritance_yaml(): - test_file = os.path.join("tests", "fixtures", "child.yaml") - data = load_data(os.path.join("tests", "fixtures", "config"), test_file) + test_file = fs.path.join("tests", "fixtures", "child.yaml") + data = load_data(fs.path.join("tests", "fixtures", "config"), test_file) eq_(data, {"key": "hello world", "pass": "ox"}) @raises(IOError) def test_exception(): - test_file = os.path.join("tests", "fixtures", "orphan.yaml") - load_data(os.path.join("tests", "fixtures", "config"), test_file) + test_file = fs.path.join("tests", "fixtures", "orphan.yaml") + load_data(fs.path.join("tests", "fixtures", "config"), test_file) @raises(IOError) def test_exception_2(): - test_file = os.path.join("tests", "fixtures", "dragon.yaml") - load_data(os.path.join("tests", "fixtures", "config"), test_file) + test_file = fs.path.join("tests", "fixtures", "dragon.yaml") + load_data(fs.path.join("tests", "fixtures", "config"), test_file) @raises(IOError) def test_exception_3(): - test_file = os.path.join("tests", "fixtures", "dragon.yaml") + test_file = fs.path.join("tests", "fixtures", "dragon.yaml") load_data(None, test_file) From b921ffe06183a43540d40228702e280ea8391006 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 09:05:08 +0100 Subject: [PATCH 012/167] :green_heart: upgrade engine tests --- tests/test_engine.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 803258fa..7c503aba 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,4 +1,5 @@ import os +import fs.path import sys import moban.exceptions as exceptions @@ -14,14 +15,14 @@ from moban.plugins.context import Context from moban.plugins.template import MobanEngine, expand_template_directories -USER_HOME = os.path.join("user", "home", ".moban", "repos") +USER_HOME = fs.path.join("user", "home", ".moban", "repos") @PluginInfo("library", tags=["testmobans"]) class TestPypkg: def __init__(self): - __package_path__ = os.path.dirname(__file__) - self.resources_path = os.path.join(__package_path__, "fixtures") + __package_path__ = fs.path.dirname(__file__) + self.resources_path = fs.path.join(__package_path__, "fixtures") def test_expand_pypi_dir(): @@ -35,7 +36,7 @@ def test_expand_pypi_dir(): def test_expand_repo_dir(_, __): dirs = list(expand_template_directories("git_repo:template")) - expected = [os.path.join(USER_HOME, "git_repo", "template")] + expected = [fs.path.join(USER_HOME, "git_repo", "template")] eq_(expected, dirs) @@ -77,7 +78,7 @@ def test_non_existent_ctx_directries(): def test_file_tests(): output = "test.txt" - path = os.path.join("tests", "fixtures", "jinja_tests") + path = fs.path.join("tests", "fixtures", "jinja_tests") engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("file_tests.template", "file_tests.yml", output) with open(output, "r") as output_file: @@ -88,7 +89,7 @@ def test_file_tests(): def test_global_template_variables(): output = "test.txt" - path = os.path.join("tests", "fixtures", "globals") + path = fs.path.join("tests", "fixtures", "globals") engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("variables.template", "variables.yml", output) with open(output, "r") as output_file: @@ -99,7 +100,7 @@ def test_global_template_variables(): def test_nested_global_template_variables(): output = "test.txt" - path = os.path.join("tests", "fixtures", "globals") + path = fs.path.join("tests", "fixtures", "globals") engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("nested.template", "variables.yml", output) with open(output, "r") as output_file: @@ -113,7 +114,7 @@ def test_environ_variables_as_data(): test_value = "foo" os.environ[test_var] = test_value output = "test.txt" - path = os.path.join("tests", "fixtures", "environ_vars_as_data") + path = fs.path.join("tests", "fixtures", "environ_vars_as_data") engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("test.template", "this_does_not_exist.yml", output) with open(output, "r") as output_file: @@ -124,7 +125,7 @@ def test_environ_variables_as_data(): def test_string_template(): output = "test.txt" - path = os.path.join("tests", "fixtures") + path = fs.path.join("tests", "fixtures") engine = ENGINES.get_engine("jinja2", [path], path) engine.render_string_to_file("{{simple}}", "simple.yaml", output) with open(output, "r") as output_file: From e4f17766e83e2fb238ccc59ab8784187e34988b4 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 09:11:48 +0100 Subject: [PATCH 013/167] :hammer: more replacement of os.path -> fs.path --- tests/mobanfile/test_mobanfile.py | 10 ++++++---- tests/mobanfile/test_templates.py | 10 +++++----- tests/test_copy_encoding.py | 6 +++--- tests/test_engine.py | 3 ++- tests/test_utils.py | 16 +++++++++------- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/tests/mobanfile/test_mobanfile.py b/tests/mobanfile/test_mobanfile.py index 4592c76c..6a0214bd 100644 --- a/tests/mobanfile/test_mobanfile.py +++ b/tests/mobanfile/test_mobanfile.py @@ -4,6 +4,8 @@ from nose.tools import eq_ from moban.definitions import GitRequire, TemplateTarget +import fs.path + class TestFinder: def setUp(self): @@ -118,14 +120,14 @@ def test_handle_targets(fake_renderer): TEMPLATE = "copier-test01.csv" OUTPUT = "output.csv" CONFIGURATION = "child.yaml" - TEMPLATE_DIRS = [os.path.join("tests", "fixtures")] + TEMPLATE_DIRS = [fs.path.join("tests", "fixtures")] DEFAULT_TEMPLATE_TYPE = "jinja2" options = dict( configuration=CONFIGURATION, template_type=DEFAULT_TEMPLATE_TYPE, template_dir=TEMPLATE_DIRS, - configuration_dir=os.path.join("tests", "fixtures"), + configuration_dir=fs.path.join("tests", "fixtures"), ) short_hand_targets = [{OUTPUT: TEMPLATE}] handle_targets(options, short_hand_targets) @@ -152,14 +154,14 @@ def test_handle_targets_sequence(fake_renderer): OUTPUT1 = "filterme.handlebars" # in the future, this could dynamic output OUTPUT2 = "filtered_output.txt" CONFIGURATION = "child.yaml" - TEMPLATE_DIRS = [os.path.join("tests", "fixtures", "mobanfile")] + TEMPLATE_DIRS = [fs.path.join("tests", "fixtures", "mobanfile")] DEFAULT_TEMPLATE_TYPE = "jinja2" options = dict( configuration=CONFIGURATION, template_type=DEFAULT_TEMPLATE_TYPE, template_dir=TEMPLATE_DIRS, - configuration_dir=os.path.join("tests", "fixtures"), + configuration_dir=fs.path.join("tests", "fixtures"), ) short_hand_targets = [{OUTPUT1: TEMPLATE1}, {OUTPUT2: OUTPUT1}] handle_targets(options, short_hand_targets) diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index b8cf1e8c..5f07d9c7 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -37,7 +37,7 @@ def test_listing_dir(self): expected = [ ( "copier-directory/level1-file1", - os.path.join("/tmp/copy-a-directory", "level1-file1"), + fs.path.join("/tmp/copy-a-directory", "level1-file1"), None, ) ] @@ -50,15 +50,15 @@ def test_listing_dir_recusively(self): ) expected = [ ( - os.path.join("copier-directory", "copier-sample-dir", "file1"), - os.path.join( + fs.path.join("copier-directory", "copier-sample-dir", "file1"), + fs.path.join( "/tmp/copy-a-directory", "copier-sample-dir", "file1" ), None, ), ( - os.path.join("copier-directory", "level1-file1"), - os.path.join("/tmp/copy-a-directory", "level1-file1"), + fs.path.join("copier-directory", "level1-file1"), + fs.path.join("/tmp/copy-a-directory", "level1-file1"), None, ), ] diff --git a/tests/test_copy_encoding.py b/tests/test_copy_encoding.py index de45bfbe..50880ef8 100644 --- a/tests/test_copy_encoding.py +++ b/tests/test_copy_encoding.py @@ -1,12 +1,12 @@ -import os - from moban.copy import ContentForwardEngine from nose.tools import eq_ +import fs.path + class TestCopyEncoding: def setUp(self): - template_path = os.path.join("tests", "fixtures") + template_path = fs.path.join("tests", "fixtures") self.engine = ContentForwardEngine([template_path]) def test_encoding_of_template(self): diff --git a/tests/test_engine.py b/tests/test_engine.py index 7c503aba..8be38606 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,5 +1,4 @@ import os -import fs.path import sys import moban.exceptions as exceptions @@ -15,6 +14,8 @@ from moban.plugins.context import Context from moban.plugins.template import MobanEngine, expand_template_directories +import fs.path + USER_HOME = fs.path.join("user", "home", ".moban", "repos") diff --git a/tests/test_utils.py b/tests/test_utils.py index 7ce9ffcf..5c189800 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -15,6 +15,8 @@ ) from moban.exceptions import FileNotFound +import fs.path + def create_file(test_file, permission): with open(test_file, "w") as f: @@ -96,8 +98,8 @@ def test_mkdir_p(): @raises(FileNotFound) def test_get_template_path_with_error(): temp_dirs = [ - os.path.join("tests", "fixtures", "template-tests"), - os.path.join("tests", "abc"), + fs.path.join("tests", "fixtures", "template-tests"), + fs.path.join("tests", "abc"), ] template = "I-do-not-exist.jj2" get_template_path(temp_dirs, template) @@ -105,15 +107,15 @@ def test_get_template_path_with_error(): def test_get_template_path(): temp_dirs = [ - os.path.join("tests", "fixtures", "template-tests"), - os.path.join("tests", "abc"), - os.path.join("tests", "abc"), + fs.path.join("tests", "fixtures", "template-tests"), + fs.path.join("tests", "abc"), + fs.path.join("tests", "abc"), ] template = "a.jj2" template_path = get_template_path(temp_dirs, template) - expected = os.path.join( + expected = fs.path.join( os.getcwd(), - os.path.join("tests", "fixtures", "template-tests", "a.jj2"), + fs.path.join("tests", "fixtures", "template-tests", "a.jj2"), ) eq_(template_path, expected) From 52b1773956f934eb4bf3d2f2d02c33400b2575d5 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 18:55:59 +0100 Subject: [PATCH 014/167] :hammer: update fs layer --- moban/fs.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ moban/utils.py | 3 ++- 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 moban/fs.py diff --git a/moban/fs.py b/moban/fs.py new file mode 100644 index 00000000..76517783 --- /dev/null +++ b/moban/fs.py @@ -0,0 +1,45 @@ +import sys + +import fs.path + +PY2 = sys.version_info[0] == 2 + + +def is_dir(path): + path = to_unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as the_fs: + return the_fs.isdir(the_file_name) + + +def is_file(path): + path = to_unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as the_fs: + return the_fs.isfile(the_file_name) + + +def exists(path): + path = to_unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as the_fs: + return the_fs.exists(the_file_name) + + +def list_dir(path): + path = to_unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as fs_system: + for file_name in fs_system.listdir(the_file_name): + yield file_name + + +def to_unicode(path): + if PY2: + if isinstance(path, unicode) is False: + return unicode(path) + return path diff --git a/moban/utils.py b/moban/utils.py index d8435ee3..7c55093b 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -5,6 +5,7 @@ import logging from moban import constants, exceptions +from moban import fs as moban_fs import fs from fs import path as fs_path @@ -140,7 +141,7 @@ def find_file_in_template_dirs(src, template_dirs): log.debug(template_dirs) for folder in template_dirs: with fs.open_fs(folder) as fs_system: - if fs_system.exists(src): + if fs_system.exists(moban_fs.to_unicode(src)): return fs_system.getsyspath(src) else: return None From 4822b83b4a19a573d9c6ff173b95f80d6806b75b Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 18:56:45 +0100 Subject: [PATCH 015/167] :hammer: bank the template refactoring change --- moban/mobanfile/templates.py | 82 +++++++++++------------------------- 1 file changed, 24 insertions(+), 58 deletions(-) diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 27b38648..422f2eef 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -1,7 +1,6 @@ -import os -import sys import logging +from moban import fs as moban_fs from moban import reporter from moban.utils import find_file_in_template_dirs @@ -9,27 +8,6 @@ import fs.path log = logging.getLogger(__name__) -PY2 = sys.version_info[0] == 2 - - -def is_dir(path): - if PY2: - if isinstance(path, unicode) is False: - path = unicode(path) - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - with fs.open_fs(dir_name) as the_fs: - return the_fs.isdir(the_file_name) - - -def is_file(path): - if PY2: - if isinstance(path, unicode) is False: - path = unicode(path) - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - with fs.open_fs(dir_name) as the_fs: - return the_fs.isfile(the_file_name) def handle_template(template_file, output, template_dirs): @@ -54,7 +32,7 @@ def handle_template(template_file, output, template_dirs): reporter.report_error_message( "{0} cannot be found".format(template_file) ) - elif is_dir(template_file_on_disk): + elif moban_fs.is_dir(template_file_on_disk): for a_triple in _list_dir_files( template_file, template_file_on_disk, output ): @@ -65,44 +43,32 @@ def handle_template(template_file, output, template_dirs): def _list_dir_files(source, actual_source_path, dest): - if PY2: - if isinstance(path, unicode) is False: - actual_source_path = unicode(actual_source_path) - dir_name = fs.path.dirname(actual_source_path) - the_file_name = fs.path.basename(actual_source_path) - with fs.open_fs(dir_name) as fs_system: - for file_name in fs_system.listdir(the_file_name): - if fs_system.isfile(fs.path.join(the_file_name, file_name)): - # please note jinja2 does NOT like windows path - # hence the following statement looks like cross platform - # src_file_under_dir = os.path.join(source, file_name) - # but actually it breaks windows instead. - src_file_under_dir = "%s/%s" % (source, file_name) + for file_name in moban_fs.list_dir(actual_source_path): + if moban_fs.is_file(fs.path.join(actual_source_path, file_name)): + # please note jinja2 does NOT like windows path + # hence the following statement looks like cross platform + # src_file_under_dir = os.path.join(source, file_name) + # but actually it breaks windows instead. + src_file_under_dir = "%s/%s" % (source, file_name) - dest_file_under_dir = fs.path.join(dest, file_name) - template_type = _get_template_type(src_file_under_dir) - yield (src_file_under_dir, dest_file_under_dir, template_type) + dest_file_under_dir = fs.path.join(dest, file_name) + template_type = _get_template_type(src_file_under_dir) + yield (src_file_under_dir, dest_file_under_dir, template_type) def _listing_directory_files_recusively(source, actual_source_path, dest): - if PY2: - if isinstance(path, unicode) is False: - actual_source_path = unicode(actual_source_path) - dir_name = fs.path.dirname(actual_source_path) - the_file_name = fs.path.basename(actual_source_path) - with fs.open_fs(dir_name) as fs_system: - for file_name in fs_system.listdir(the_file_name): - src_file_under_dir = fs.path.join(source, file_name) - dest_file_under_dir = fs.path.join(dest, file_name) - real_src_file = fs.path.join(actual_source_path, file_name) - if fs_system.isfile(fs.path.join(the_file_name, file_name)): - template_type = _get_template_type(src_file_under_dir) - yield (src_file_under_dir, dest_file_under_dir, template_type) - elif fs_system.isdir(fs.path.join(the_file_name, file_name)): - for a_triple in _listing_directory_files_recusively( - src_file_under_dir, real_src_file, dest_file_under_dir - ): - yield a_triple + for file_name in moban_fs.list_dir(actual_source_path): + src_file_under_dir = fs.path.join(source, file_name) + dest_file_under_dir = fs.path.join(dest, file_name) + real_src_file = fs.path.join(actual_source_path, file_name) + if moban_fs.is_file(fs.path.join(actual_source_path, file_name)): + template_type = _get_template_type(src_file_under_dir) + yield (src_file_under_dir, dest_file_under_dir, template_type) + elif moban_fs.is_dir(fs.path.join(actual_source_path, file_name)): + for a_triple in _listing_directory_files_recusively( + src_file_under_dir, real_src_file, dest_file_under_dir + ): + yield a_triple def _get_template_type(template_file): From b78e655486d389cfa9fdabb9d000d918fe42c00c Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 19:09:21 +0100 Subject: [PATCH 016/167] :green_heart: all existing tests now pass --- moban/fs.py | 8 ++++++++ moban/plugins/template.py | 14 +++++++++----- .../integration_tests/test_command_line_options.py | 3 ++- tests/test_engine.py | 2 +- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/moban/fs.py b/moban/fs.py index 76517783..ae42a89d 100644 --- a/moban/fs.py +++ b/moban/fs.py @@ -38,6 +38,14 @@ def list_dir(path): yield file_name +def abspath(path): + path = to_unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as the_fs: + return the_fs.getsyspath(the_file_name) + + def to_unicode(path): if PY2: if isinstance(path, unicode) is False: diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 8a72f9e7..8ba89472 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -1,4 +1,5 @@ import os +import fs.path import sys import logging @@ -8,6 +9,7 @@ from moban.plugins.context import Context from moban.plugins.library import LIBRARIES from moban.plugins.strategy import Strategy +from moban import fs as moban_fs log = logging.getLogger(__name__) PY3_ABOVE = sys.version_info[0] > 2 @@ -179,16 +181,17 @@ def expand_template_directories(dirs): def expand_template_directory(directory): log.debug("Expanding %s..." % directory) + translated_directory = None if ":" in directory and directory[1] != ":": library_or_repo_name, relative_path = directory.split(":") - potential_repo_path = os.path.join( + potential_repo_path = fs.path.join( repo.get_moban_home(), library_or_repo_name ) - if os.path.exists(potential_repo_path): + if moban_fs.exists(potential_repo_path): # expand repo template path if relative_path: - translated_directory = os.path.join( + translated_directory = fs.path.join( potential_repo_path, relative_path ) else: @@ -197,12 +200,13 @@ def expand_template_directory(directory): # expand pypi template path library_path = LIBRARIES.resource_path_of(library_or_repo_name) if relative_path: - translated_directory = os.path.join( + translated_directory = fs.path.join( library_path, relative_path ) else: translated_directory = library_path else: # local template path - translated_directory = os.path.abspath(directory) + print(directory) + translated_directory = moban_fs.abspath(directory) return translated_directory diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index 2c141bf6..76b219f3 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -17,8 +17,9 @@ def setUp(self): ) self.patcher1.start() + @patch("moban.fs.abspath") @patch("moban.plugins.template.MobanEngine.render_to_file") - def test_custom_options(self, fake_template_doer): + def test_custom_options(self, fake_template_doer, fake_abspath): test_args = [ "moban", "-c", diff --git a/tests/test_engine.py b/tests/test_engine.py index 8be38606..1414a047 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -33,7 +33,7 @@ def test_expand_pypi_dir(): @patch("moban.repo.get_moban_home", return_value=USER_HOME) -@patch("os.path.exists", return_value=True) +@patch("moban.fs.exists", return_value=True) def test_expand_repo_dir(_, __): dirs = list(expand_template_directories("git_repo:template")) From 45bf54affd33cb212a5650f9f0e52f710056be1d Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 19:10:27 +0100 Subject: [PATCH 017/167] :sparkles: update constants.py --- moban/constants.py | 6 +++--- moban/plugins/template.py | 5 +++-- moban/utils.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/moban/constants.py b/moban/constants.py index 507ee903..df476b0d 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -1,4 +1,4 @@ -import os +import fs.path # Template type TEMPLATE_JINJA2 = "jinja2" @@ -41,9 +41,9 @@ DEFAULT_TEMPLATE_DIRNAME = ".%s.td" % PROGRAM_NAME DEFAULT_OPTIONS = { # .moban.cd, default configuration dir - LABEL_CONFIG_DIR: os.path.join(".", DEFAULT_CONFIGURATION_DIRNAME), + LABEL_CONFIG_DIR: fs.path.join(".", DEFAULT_CONFIGURATION_DIRNAME), # .moban.td, default template dirs - LABEL_TMPL_DIRS: [".", os.path.join(".", DEFAULT_TEMPLATE_DIRNAME)], + LABEL_TMPL_DIRS: [".", fs.path.join(".", DEFAULT_TEMPLATE_DIRNAME)], # moban.output, default output file name LABEL_OUTPUT: "%s.output" % PROGRAM_NAME, # data.yml, default data input file diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 8ba89472..14af4dc7 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -1,15 +1,16 @@ import os -import fs.path import sys import logging +from moban import fs as moban_fs from moban import repo, utils, reporter, constants, exceptions from lml.plugin import PluginManager from moban.hashstore import HASH_STORE from moban.plugins.context import Context from moban.plugins.library import LIBRARIES from moban.plugins.strategy import Strategy -from moban import fs as moban_fs + +import fs.path log = logging.getLogger(__name__) PY3_ABOVE = sys.version_info[0] > 2 diff --git a/moban/utils.py b/moban/utils.py index 7c55093b..39ee916e 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -4,8 +4,8 @@ import errno import logging -from moban import constants, exceptions from moban import fs as moban_fs +from moban import constants, exceptions import fs from fs import path as fs_path From 46a018ddd1d8eb53525931798eb9487f43ef8f8e Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 23 Jul 2019 22:47:23 +0100 Subject: [PATCH 018/167] :green_heart: fix linux unit test failures --- tests/integration_tests/test_command_line_options.py | 9 ++++++--- tests/mobanfile/test_mobanfile.py | 2 -- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index 76b219f3..9db5b68b 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -160,8 +160,9 @@ def test_single_command_with_missing_output(self, fake_template_doer): main() + @patch("moban.fs.exists", return_value=True) @patch("moban.plugins.template.MobanEngine.render_to_files") - def test_single_command_with_a_few_options(self, fake_template_doer): + def test_single_command_with_a_few_options(self, fake_template_doer, _): test_args = ["moban", "-t", "README.rst.jj2", "-o", "xyz.output"] with patch.object(sys, "argv", test_args): from moban.main import main @@ -261,8 +262,9 @@ def setUp(self): ) self.patcher1.start() + @patch("moban.fs.exists", return_value=True) @patch("moban.plugins.template.MobanEngine.render_to_files") - def test_single_command(self, fake_template_doer): + def test_single_command(self, fake_template_doer, _): test_args = ["moban", "-m", self.config_file] with patch.object(sys, "argv", test_args): from moban.main import main @@ -386,10 +388,11 @@ def setUp(self): with open(self.data_file, "w") as f: f.write("hello: world") + @patch("moban.fs.exists", return_value=True) @patch( "moban.utils.verify_the_existence_of_directories", return_value=True ) - def test_single_command(self, _): + def test_single_command(self, _, __): test_args = ["moban"] with patch.object(sys, "argv", test_args): from moban.main import main diff --git a/tests/mobanfile/test_mobanfile.py b/tests/mobanfile/test_mobanfile.py index 6a0214bd..321b8e2f 100644 --- a/tests/mobanfile/test_mobanfile.py +++ b/tests/mobanfile/test_mobanfile.py @@ -1,5 +1,3 @@ -import os - from mock import patch from nose.tools import eq_ from moban.definitions import GitRequire, TemplateTarget From dc86afb139b091ec8723b22904ae950aabdcd100 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 00:12:18 +0100 Subject: [PATCH 019/167] :green_heart: further fight with unit tests --- moban/{fs.py => file_system.py} | 8 ++++++-- moban/mobanfile/__init__.py | 1 + moban/mobanfile/templates.py | 2 +- moban/plugins/template.py | 3 +-- moban/utils.py | 10 +++++----- tests/integration_tests/test_command_line_options.py | 12 +++++------- tests/test_engine.py | 2 +- 7 files changed, 20 insertions(+), 18 deletions(-) rename moban/{fs.py => file_system.py} (87%) diff --git a/moban/fs.py b/moban/file_system.py similarity index 87% rename from moban/fs.py rename to moban/file_system.py index ae42a89d..a4f97e37 100644 --- a/moban/fs.py +++ b/moban/file_system.py @@ -1,5 +1,6 @@ import sys +import fs import fs.path PY2 = sys.version_info[0] == 2 @@ -25,8 +26,11 @@ def exists(path): path = to_unicode(path) dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) - with fs.open_fs(dir_name) as the_fs: - return the_fs.exists(the_file_name) + try: + with fs.open_fs(dir_name) as the_fs: + return the_fs.exists(the_file_name) + except fs.errors.CreateFailed: + return False def list_dir(path): diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index 2d3dca4e..c125ea9f 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -132,6 +132,7 @@ def handle_targets(merged_options, targets): jobs_for_each_engine[primary_template_type].append(target) count = 0 + for template_type in jobs_for_each_engine.keys(): engine = plugins.ENGINES.get_engine( template_type, diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 422f2eef..aabcdd9d 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -1,7 +1,7 @@ import logging -from moban import fs as moban_fs from moban import reporter +from moban import file_system as moban_fs from moban.utils import find_file_in_template_dirs import fs diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 14af4dc7..34282485 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -1,9 +1,8 @@ -import os import sys import logging -from moban import fs as moban_fs from moban import repo, utils, reporter, constants, exceptions +from moban import file_system as moban_fs from lml.plugin import PluginManager from moban.hashstore import HASH_STORE from moban.plugins.context import Context diff --git a/moban/utils.py b/moban/utils.py index 39ee916e..0c7389b8 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -4,12 +4,12 @@ import errno import logging -from moban import fs as moban_fs from moban import constants, exceptions +from moban import file_system as moban_fs import fs from fs import path as fs_path -from fs import open_fs +from fs import errors, open_fs log = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -140,8 +140,8 @@ def verify_the_existence_of_directories(dirs): def find_file_in_template_dirs(src, template_dirs): log.debug(template_dirs) for folder in template_dirs: - with fs.open_fs(folder) as fs_system: - if fs_system.exists(moban_fs.to_unicode(src)): - return fs_system.getsyspath(src) + path = fs.path.join(folder, src) + if moban_fs.exists(path): + return path else: return None diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index 9db5b68b..abb57c34 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -17,7 +17,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.fs.abspath") + @patch("moban.file_system.abspath") @patch("moban.plugins.template.MobanEngine.render_to_file") def test_custom_options(self, fake_template_doer, fake_abspath): test_args = [ @@ -160,14 +160,14 @@ def test_single_command_with_missing_output(self, fake_template_doer): main() - @patch("moban.fs.exists", return_value=True) @patch("moban.plugins.template.MobanEngine.render_to_files") - def test_single_command_with_a_few_options(self, fake_template_doer, _): + def test_single_command_with_a_few_options(self, fake_template_doer): test_args = ["moban", "-t", "README.rst.jj2", "-o", "xyz.output"] with patch.object(sys, "argv", test_args): from moban.main import main main() + call_args = list(fake_template_doer.call_args[0][0]) eq_( call_args, @@ -262,9 +262,8 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.fs.exists", return_value=True) @patch("moban.plugins.template.MobanEngine.render_to_files") - def test_single_command(self, fake_template_doer, _): + def test_single_command(self, fake_template_doer): test_args = ["moban", "-m", self.config_file] with patch.object(sys, "argv", test_args): from moban.main import main @@ -388,11 +387,10 @@ def setUp(self): with open(self.data_file, "w") as f: f.write("hello: world") - @patch("moban.fs.exists", return_value=True) @patch( "moban.utils.verify_the_existence_of_directories", return_value=True ) - def test_single_command(self, _, __): + def test_single_command(self, _): test_args = ["moban"] with patch.object(sys, "argv", test_args): from moban.main import main diff --git a/tests/test_engine.py b/tests/test_engine.py index 1414a047..0d9443e2 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -33,7 +33,7 @@ def test_expand_pypi_dir(): @patch("moban.repo.get_moban_home", return_value=USER_HOME) -@patch("moban.fs.exists", return_value=True) +@patch("moban.file_system.exists", return_value=True) def test_expand_repo_dir(_, __): dirs = list(expand_template_directories("git_repo:template")) From cdabe6e1738cec223dce6aac6dee892109804675 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 00:20:21 +0100 Subject: [PATCH 020/167] :fire: remove os.path.join --- moban/utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 0c7389b8..7834bfc6 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -37,10 +37,10 @@ def merge(left, right): def search_file(base_dir, file_name): the_file = file_name - if not os.path.exists(the_file): + if not moban_fs.exists(the_file): if base_dir: - the_file = os.path.join(base_dir, file_name) - if not os.path.exists(the_file): + the_file = fs.path.join(base_dir, file_name) + if not moban_fs.exists(the_file): raise IOError( constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) ) @@ -104,14 +104,14 @@ def get_template_path(template_dirs, template): temp_dir = "" for a_dir in template_dirs: - template_file_exists = os.path.exists( - os.path.join(a_dir, template) - ) and os.path.isfile(os.path.join(a_dir, template)) + template_file_exists = moban_fs.exists( + fs.path.join(a_dir, template) + ) and moban_fs.is_file(fs.path.join(a_dir, template)) if template_file_exists: temp_dir = a_dir - temp_file_path = os.path.join( - os.getcwd(), os.path.join(temp_dir, template) + temp_file_path = fs.path.join( + os.getcwd(), fs.path.join(temp_dir, template) ) return temp_file_path raise exceptions.FileNotFound From b86fc71b7cc8929c686e90703966e7c8e8f9b8a1 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 08:47:39 +0100 Subject: [PATCH 021/167] :green_heart: remore os.path.join in repo.py and :hammer: code refactor templates.py, fs2 does not like ** --- moban/mobanfile/templates.py | 47 ++++++++++++++++++------------------ moban/repo.py | 11 +++++---- tests/test_repo.py | 2 +- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index aabcdd9d..cb8a9a16 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -12,34 +12,35 @@ def handle_template(template_file, output, template_dirs): log.info("handling %s" % template_file) - template_file_on_disk = find_file_in_template_dirs( - template_file, template_dirs - ) - if template_file_on_disk is None: - if template_file.endswith("**"): - source_dir = template_file[:-3] - src_path = find_file_in_template_dirs(source_dir, template_dirs) - if src_path: - for a_triple in _listing_directory_files_recusively( - source_dir, src_path, output - ): - yield a_triple - else: - reporter.report_error_message( - "{0} cannot be found".format(template_file) - ) + if template_file.endswith("**"): + source_dir = template_file[:-3] + src_path = find_file_in_template_dirs(source_dir, template_dirs) + if src_path: + for a_triple in _listing_directory_files_recusively( + source_dir, src_path, output + ): + yield a_triple else: reporter.report_error_message( "{0} cannot be found".format(template_file) ) - elif moban_fs.is_dir(template_file_on_disk): - for a_triple in _list_dir_files( - template_file, template_file_on_disk, output - ): - yield a_triple else: - template_type = _get_template_type(template_file) - yield (template_file, output, template_type) + template_file_on_disk = find_file_in_template_dirs( + template_file, template_dirs + ) + + if template_file_on_disk is None: + reporter.report_error_message( + "{0} cannot be found".format(template_file) + ) + elif moban_fs.is_dir(template_file_on_disk): + for a_triple in _list_dir_files( + template_file, template_file_on_disk, output + ): + yield a_triple + else: + template_type = _get_template_type(template_file) + yield (template_file, output, template_type) def _list_dir_files(source, actual_source_path, dest): diff --git a/moban/repo.py b/moban/repo.py index 02fdd6de..1e4f3500 100644 --- a/moban/repo.py +++ b/moban/repo.py @@ -1,10 +1,11 @@ -import os import sys import subprocess -from moban import reporter, constants, exceptions +from moban import reporter, constants, exceptions, file_system from moban.utils import mkdir_p +import fs.path + def git_clone(requires): from git import Repo @@ -20,8 +21,8 @@ def git_clone(requires): for require in requires: repo_name = get_repo_name(require.git_url) - local_repo_folder = os.path.join(moban_home, repo_name) - if os.path.exists(local_repo_folder): + local_repo_folder = fs.path.join(moban_home, repo_name) + if file_system.exists(local_repo_folder): reporter.report_git_pull(repo_name) repo = Repo(local_repo_folder) repo.git.pull() @@ -60,7 +61,7 @@ def get_moban_home(): from appdirs import user_cache_dir home_dir = user_cache_dir(appname=constants.PROGRAM_NAME) - return os.path.join(home_dir, constants.MOBAN_REPOS_DIR_NAME) + return fs.path.join(home_dir, constants.MOBAN_REPOS_DIR_NAME) def make_sure_git_is_available(): diff --git a/tests/test_repo.py b/tests/test_repo.py index 1c82e258..edf3d6c0 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -14,7 +14,7 @@ @patch("appdirs.user_cache_dir", return_value="root") @patch("moban.utils.mkdir_p") -@patch("os.path.exists") +@patch("moban.file_system.exists") @patch("git.Repo", autospec=True) class TestGitFunctions: def setUp(self): From ba734b57f50c70362dbad2cbb4f01777bfb2a3ae Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 08:50:41 +0100 Subject: [PATCH 022/167] :hammer: pin down lowerest version of fs --- .moban.cd/moban.yml | 2 +- min_requirements.txt | 2 +- moban/plugins/template.py | 1 - requirements.txt | 2 +- setup.py | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index fed0c7f6..4136ed87 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -28,7 +28,7 @@ dependencies: - crayons>= 0.1.0 - GitPython>=2.0.0 - git-url-parse>=1.2.2 - - fs + - fs>=2.0.12 description: Yet another jinja2 cli command for static text generation scm_host: github.com lint_command: make install_test format git-diff-check lint diff --git a/min_requirements.txt b/min_requirements.txt index fe5d153d..155bc850 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -8,4 +8,4 @@ appdirs==1.2.0 crayons== 0.1.0 GitPython==2.0.0 git-url-parse==1.2.2 -fs +fs==2.0.12 diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 34282485..64f44ae9 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -207,6 +207,5 @@ def expand_template_directory(directory): translated_directory = library_path else: # local template path - print(directory) translated_directory = moban_fs.abspath(directory) return translated_directory diff --git a/requirements.txt b/requirements.txt index 481bf3da..27e772e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ appdirs>=1.2.0 crayons>= 0.1.0 GitPython>=2.0.0 git-url-parse>=1.2.2 -fs +fs>=2.0.12 diff --git a/setup.py b/setup.py index 5dbdc995..f1a97b21 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ "crayons>= 0.1.0", "GitPython>=2.0.0", "git-url-parse>=1.2.2", - "fs", + "fs>=2.0.12", ] SETUP_COMMANDS = {} From a7f6349b344e60f6f987bacd6f5347af3b105d53 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 08:55:00 +0100 Subject: [PATCH 023/167] :hammer: more replacement in tests --- tests/test_json_loader.py | 5 ++--- tests/test_repo.py | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_json_loader.py b/tests/test_json_loader.py index 8a05ef9a..2e295efc 100644 --- a/tests/test_json_loader.py +++ b/tests/test_json_loader.py @@ -1,10 +1,9 @@ -import os - from nose.tools import eq_ from moban.data_loaders.json_loader import open_json +import fs.path def test_open_json(): - content = open_json(os.path.join("tests", "fixtures", "child.json")) + content = open_json(fs.path.join("tests", "fixtures", "child.json")) expected = {"key": "hello world", "pass": "ox"} eq_(expected, content) diff --git a/tests/test_repo.py b/tests/test_repo.py index edf3d6c0..08aeb1d7 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -10,6 +10,7 @@ from nose.tools import eq_, raises from moban.exceptions import NoGitCommand from moban.definitions import GitRequire +import fs.path @patch("appdirs.user_cache_dir", return_value="root") @@ -30,7 +31,7 @@ def setUp(self): self.require_with_reference = GitRequire( git_url=self.repo, reference="a-commit-reference" ) - self.expected_local_repo_path = os.path.join( + self.expected_local_repo_path = fs.path.join( "root", "repos", self.repo_name ) @@ -152,7 +153,7 @@ def test_get_repo_name_can_handle_invalid_url(fake_reporter): @patch("appdirs.user_cache_dir", return_value="root") def test_get_moban_home(_): actual = get_moban_home() - eq_(os.path.join("root", "repos"), actual) + eq_(fs.path.join("root", "repos"), actual) @raises(NoGitCommand) From b8033ad529d667c33fc52caf704b1eb6d5be7fc1 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 19:02:52 +0100 Subject: [PATCH 024/167] :sparkles: use migrated moban-handlebars for testing --- rnd_requirements.txt | 2 ++ tests/test_engine.py | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 rnd_requirements.txt diff --git a/rnd_requirements.txt b/rnd_requirements.txt new file mode 100644 index 00000000..0565126f --- /dev/null +++ b/rnd_requirements.txt @@ -0,0 +1,2 @@ +https://github.com/moremoban/moban-handlebars/archive/dev.zip + diff --git a/tests/test_engine.py b/tests/test_engine.py index 0d9443e2..b60fdf94 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -23,12 +23,14 @@ class TestPypkg: def __init__(self): __package_path__ = fs.path.dirname(__file__) + print(__package_path__) self.resources_path = fs.path.join(__package_path__, "fixtures") def test_expand_pypi_dir(): dirs = list(expand_template_directories("testmobans:template-tests")) for directory in dirs: + print(directory) assert os.path.exists(directory) From 714ed5b801ede0abf26fee3b4a94e443c4c31eef Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 22:39:04 +0100 Subject: [PATCH 025/167] :bug: use normcase() to get backward slashes --- moban/utils.py | 3 ++- tests/test_engine.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 7834bfc6..89f48bfa 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -111,7 +111,8 @@ def get_template_path(template_dirs, template): if template_file_exists: temp_dir = a_dir temp_file_path = fs.path.join( - os.getcwd(), fs.path.join(temp_dir, template) + os.path.normcase(os.getcwd()), + fs.path.join(temp_dir, template) ) return temp_file_path raise exceptions.FileNotFound diff --git a/tests/test_engine.py b/tests/test_engine.py index b60fdf94..44e3690f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -22,9 +22,9 @@ @PluginInfo("library", tags=["testmobans"]) class TestPypkg: def __init__(self): - __package_path__ = fs.path.dirname(__file__) + __package_path__ = os.path.normcase(os.path.dirname(__file__)) print(__package_path__) - self.resources_path = fs.path.join(__package_path__, "fixtures") + self.resources_path = os.path.join(__package_path__, "fixtures") def test_expand_pypi_dir(): From 2385a02c3b8a405459940eabff6cab2c4019dd06 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 22:53:08 +0100 Subject: [PATCH 026/167] :green_heart: fix final bug in windows os path --- moban/mobanfile/__init__.py | 4 +++- moban/utils.py | 3 +-- tests/test_json_loader.py | 1 + tests/test_repo.py | 1 + tests/test_utils.py | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index c125ea9f..98432afd 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -147,7 +147,9 @@ def handle_targets(merged_options, targets): def handle_plugin_dirs(plugin_dirs): for plugin_dir in plugin_dirs: - plugin_path = os.path.dirname(os.path.abspath(plugin_dir)) + plugin_path = os.path.normcase( + os.path.dirname(os.path.abspath(plugin_dir)) + ) if plugin_path not in sys.path: sys.path.append(plugin_path) pysearchre = re.compile(".py$", re.IGNORECASE) diff --git a/moban/utils.py b/moban/utils.py index 89f48bfa..192a8d1d 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -111,8 +111,7 @@ def get_template_path(template_dirs, template): if template_file_exists: temp_dir = a_dir temp_file_path = fs.path.join( - os.path.normcase(os.getcwd()), - fs.path.join(temp_dir, template) + os.path.normcase(os.getcwd()), fs.path.join(temp_dir, template) ) return temp_file_path raise exceptions.FileNotFound diff --git a/tests/test_json_loader.py b/tests/test_json_loader.py index 2e295efc..39c00e77 100644 --- a/tests/test_json_loader.py +++ b/tests/test_json_loader.py @@ -1,5 +1,6 @@ from nose.tools import eq_ from moban.data_loaders.json_loader import open_json + import fs.path diff --git a/tests/test_repo.py b/tests/test_repo.py index 08aeb1d7..c466b646 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -10,6 +10,7 @@ from nose.tools import eq_, raises from moban.exceptions import NoGitCommand from moban.definitions import GitRequire + import fs.path diff --git a/tests/test_utils.py b/tests/test_utils.py index 5c189800..e14f0614 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -114,7 +114,7 @@ def test_get_template_path(): template = "a.jj2" template_path = get_template_path(temp_dirs, template) expected = fs.path.join( - os.getcwd(), + os.path.normcase(os.getcwd()), fs.path.join("tests", "fixtures", "template-tests", "a.jj2"), ) eq_(template_path, expected) From 22cd1a8fc9b0ee774e52da636f748d44bf211def Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 23:03:48 +0100 Subject: [PATCH 027/167] :green_heart: fix the misunderstanding of fs.path.join --- moban/utils.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 192a8d1d..22e24987 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -101,19 +101,13 @@ def pip_install(packages): def get_template_path(template_dirs, template): - temp_dir = "" - for a_dir in template_dirs: template_file_exists = moban_fs.exists( fs.path.join(a_dir, template) ) and moban_fs.is_file(fs.path.join(a_dir, template)) if template_file_exists: - temp_dir = a_dir - temp_file_path = fs.path.join( - os.path.normcase(os.getcwd()), fs.path.join(temp_dir, template) - ) - return temp_file_path + return fs.path.join(a_dir, template) raise exceptions.FileNotFound From 48d0754e771a1182c73e2ffb01e04bd9a127ed42 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 23:13:47 +0100 Subject: [PATCH 028/167] :green_heart: one more attempt to fix windows behaviour --- moban/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moban/utils.py b/moban/utils.py index 22e24987..e778de7b 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -107,7 +107,8 @@ def get_template_path(template_dirs, template): ) and moban_fs.is_file(fs.path.join(a_dir, template)) if template_file_exists: - return fs.path.join(a_dir, template) + return fs.path.join(os.path.normcase(os.getcwd()), + a_dir, template) raise exceptions.FileNotFound From 98b6f8dfb7e16e769a5e785c4a1dc81faff6b458 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 23:22:09 +0100 Subject: [PATCH 029/167] :green_heart: fix the wild behavior --- moban/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index e778de7b..bb6df715 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -107,8 +107,8 @@ def get_template_path(template_dirs, template): ) and moban_fs.is_file(fs.path.join(a_dir, template)) if template_file_exists: - return fs.path.join(os.path.normcase(os.getcwd()), - a_dir, template) + return os.path.normcase(os.path.join( + os.getcwd(), a_dir, template)) raise exceptions.FileNotFound From 2481464fc5e38026fbcad0e35f23bc78679c162d Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 23:25:52 +0100 Subject: [PATCH 030/167] :green_heart: use norm path over norm case --- moban/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/utils.py b/moban/utils.py index bb6df715..e327b9e5 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -107,7 +107,7 @@ def get_template_path(template_dirs, template): ) and moban_fs.is_file(fs.path.join(a_dir, template)) if template_file_exists: - return os.path.normcase(os.path.join( + return os.path.normpath(os.path.join( os.getcwd(), a_dir, template)) raise exceptions.FileNotFound From 1f94f00c71a26bbfe435b935c6c3d2c18edfbd33 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 23:29:14 +0100 Subject: [PATCH 031/167] :sparkles: nailed the final bug --- moban/utils.py | 3 +-- tests/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index e327b9e5..49bc6a21 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -107,8 +107,7 @@ def get_template_path(template_dirs, template): ) and moban_fs.is_file(fs.path.join(a_dir, template)) if template_file_exists: - return os.path.normpath(os.path.join( - os.getcwd(), a_dir, template)) + return os.path.normpath(os.path.join(os.getcwd(), a_dir, template)) raise exceptions.FileNotFound diff --git a/tests/test_utils.py b/tests/test_utils.py index e14f0614..02a97da5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -114,7 +114,7 @@ def test_get_template_path(): template = "a.jj2" template_path = get_template_path(temp_dirs, template) expected = fs.path.join( - os.path.normcase(os.getcwd()), + os.path.normpath(os.getcwd()), fs.path.join("tests", "fixtures", "template-tests", "a.jj2"), ) eq_(template_path, expected) From 959858179855eeaa3dbe8d5e61bee3fd71753e37 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 23:34:08 +0100 Subject: [PATCH 032/167] :sparkles: python file system 2 is driving downstream code mad --- moban/utils.py | 4 +++- tests/test_utils.py | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 49bc6a21..46599109 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -107,7 +107,9 @@ def get_template_path(template_dirs, template): ) and moban_fs.is_file(fs.path.join(a_dir, template)) if template_file_exists: - return os.path.normpath(os.path.join(os.getcwd(), a_dir, template)) + return os.path.normcase( + os.path.normpath(os.path.join(os.getcwd(), a_dir, template)) + ) raise exceptions.FileNotFound diff --git a/tests/test_utils.py b/tests/test_utils.py index 02a97da5..d3b422d2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -113,9 +113,12 @@ def test_get_template_path(): ] template = "a.jj2" template_path = get_template_path(temp_dirs, template) - expected = fs.path.join( - os.path.normpath(os.getcwd()), - fs.path.join("tests", "fixtures", "template-tests", "a.jj2"), + expected = os.path.normcase( + os.path.normpath( + os.path.join( + os.getcwd(), "tests", "fixtures", "template-tests", "a.jj2" + ) + ) ) eq_(template_path, expected) From 32e097ab62d455256e4973e4d156701d01df8edc Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 23:43:23 +0100 Subject: [PATCH 033/167] :sparkles: no longer I know which file path is correct --- moban/utils.py | 4 +--- tests/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 46599109..22e24987 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -107,9 +107,7 @@ def get_template_path(template_dirs, template): ) and moban_fs.is_file(fs.path.join(a_dir, template)) if template_file_exists: - return os.path.normcase( - os.path.normpath(os.path.join(os.getcwd(), a_dir, template)) - ) + return fs.path.join(a_dir, template) raise exceptions.FileNotFound diff --git a/tests/test_utils.py b/tests/test_utils.py index d3b422d2..603841df 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -116,7 +116,7 @@ def test_get_template_path(): expected = os.path.normcase( os.path.normpath( os.path.join( - os.getcwd(), "tests", "fixtures", "template-tests", "a.jj2" + "tests", "fixtures", "template-tests", "a.jj2" ) ) ) From 283fe2be49bac83257c881f21e5db73395d91ba2 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 24 Jul 2019 23:47:48 +0100 Subject: [PATCH 034/167] :art: yes, last one should work --- tests/test_utils.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 603841df..c907f073 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -113,13 +113,7 @@ def test_get_template_path(): ] template = "a.jj2" template_path = get_template_path(temp_dirs, template) - expected = os.path.normcase( - os.path.normpath( - os.path.join( - "tests", "fixtures", "template-tests", "a.jj2" - ) - ) - ) + expected = fs.path.join("tests", "fixtures", "template-tests", "a.jj2") eq_(template_path, expected) From bb9e2611117c4871741f091ea0ab4837ff51a90a Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 25 Jul 2019 00:12:50 +0100 Subject: [PATCH 035/167] :shirt: fix lint errors --- moban/data_loaders/json_loader.py | 18 ++++-------------- moban/data_loaders/yaml.py | 21 +++++---------------- moban/file_system.py | 24 ++++++++++++++++++++++++ moban/utils.py | 15 ++++++++------- test.bat | 2 +- tests/mobanfile/test_templates.py | 2 -- 6 files changed, 42 insertions(+), 40 deletions(-) diff --git a/moban/data_loaders/json_loader.py b/moban/data_loaders/json_loader.py index 84cdadb9..15e2d4ec 100644 --- a/moban/data_loaders/json_loader.py +++ b/moban/data_loaders/json_loader.py @@ -1,12 +1,8 @@ -import sys import json from moban import constants from lml.plugin import PluginInfo - -from fs import path, open_fs - -PY2 = sys.version_info[0] == 2 +from moban.file_system import open_fs @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["json"]) @@ -14,12 +10,6 @@ def open_json(file_name): """ returns json contents as string """ - if PY2: - if isinstance(file_name, unicode) is False: - file_name = unicode(file_name) - dir_name = path.dirname(file_name) - the_file_name = path.basename(file_name) - with open_fs(dir_name) as the_fs: - with the_fs.open(the_file_name) as json_file: - data = json.load(json_file) - return data + with open_fs(file_name) as json_file: + data = json.load(json_file) + return data diff --git a/moban/data_loaders/yaml.py b/moban/data_loaders/yaml.py index f5ed4bf1..9d051f85 100644 --- a/moban/data_loaders/yaml.py +++ b/moban/data_loaders/yaml.py @@ -1,23 +1,12 @@ -import sys - from moban import constants from lml.plugin import PluginInfo from ruamel.yaml import YAML - -from fs import path, open_fs - -PY2 = sys.version_info[0] == 2 +from moban.file_system import open_fs @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["yaml", "yml"]) def open_yaml(file_name): - if PY2: - if isinstance(file_name, unicode) is False: - file_name = unicode(file_name) - dir_name = path.dirname(file_name) - the_file_name = path.basename(file_name) - with open_fs(dir_name) as the_fs: - with the_fs.open(the_file_name) as data_yaml: - yaml = YAML(typ="rt") - data = yaml.load(data_yaml) - return data + with open_fs(file_name) as data_yaml: + yaml = YAML(typ="rt") + data = yaml.load(data_yaml) + return data diff --git a/moban/file_system.py b/moban/file_system.py index a4f97e37..4c04bb21 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -2,10 +2,34 @@ import fs import fs.path +from contextlib import contextmanager + PY2 = sys.version_info[0] == 2 +@contextmanager +def open_fs(path): + path = to_unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + the_fs = fs.open_fs(dir_name) + f = the_fs.open(the_file_name) + try: + yield f + finally: + f.close() + the_fs.close() + + +def write_bytes(filename, bytes_content): + filename = to_unicode(filename) + dir_name = fs.path.dirname(filename) + the_file_name = fs.path.basename(filename) + with open_fs(dir_name) as the_fs: + the_fs.writebytes(the_file_name, bytes_content) + + def is_dir(path): path = to_unicode(path) dir_name = fs.path.dirname(path) diff --git a/moban/utils.py b/moban/utils.py index 22e24987..3e2d0a9a 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -5,11 +5,11 @@ import logging from moban import constants, exceptions -from moban import file_system as moban_fs +from moban import file_system import fs from fs import path as fs_path -from fs import errors, open_fs +from fs import open_fs log = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -37,10 +37,10 @@ def merge(left, right): def search_file(base_dir, file_name): the_file = file_name - if not moban_fs.exists(the_file): + if not file_system.exists(the_file): if base_dir: the_file = fs.path.join(base_dir, file_name) - if not moban_fs.exists(the_file): + if not file_system.exists(the_file): raise IOError( constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) ) @@ -80,6 +80,7 @@ def write_file_out(filename, content): the_file_name = fs_path.basename(filename) with open_fs(dir_name) as the_fs: the_fs.writebytes(the_file_name, content) + #file_system.write_bytes(filename, content) def mkdir_p(path): @@ -102,9 +103,9 @@ def pip_install(packages): def get_template_path(template_dirs, template): for a_dir in template_dirs: - template_file_exists = moban_fs.exists( + template_file_exists = file_system.exists( fs.path.join(a_dir, template) - ) and moban_fs.is_file(fs.path.join(a_dir, template)) + ) and file_system.is_file(fs.path.join(a_dir, template)) if template_file_exists: return fs.path.join(a_dir, template) @@ -135,7 +136,7 @@ def find_file_in_template_dirs(src, template_dirs): log.debug(template_dirs) for folder in template_dirs: path = fs.path.join(folder, src) - if moban_fs.exists(path): + if file_system.exists(path): return path else: return None diff --git a/test.bat b/test.bat index 399c6eda..e0b1ba5f 100644 --- a/test.bat +++ b/test.bat @@ -2,7 +2,7 @@ pip freeze nosetests --with-coverage --cover-package=moban --cover-package=tests || goto :error -flake8 --max-line-length=88 --exclude=docs,.moban.d --ignore=W503,W504 || goto :error +flake8 --max-line-length=88 --exclude=docs,.moban.d --ignore=W503,W504 --builtins=unicode|| goto :error :error echo Failed with error #%errorlevel%. diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index 5f07d9c7..61b69479 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -1,5 +1,3 @@ -import os - from mock import patch from nose.tools import eq_ from moban.mobanfile.templates import handle_template From fef52fcd950230448f4b1be12c32802d2705d04b Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 25 Jul 2019 00:15:32 +0100 Subject: [PATCH 036/167] :short: :hammer: code refactoring --- moban/file_system.py | 7 +++---- moban/utils.py | 14 ++------------ 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index 4c04bb21..0de9a105 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -1,9 +1,8 @@ import sys +from contextlib import contextmanager import fs import fs.path -from contextlib import contextmanager - PY2 = sys.version_info[0] == 2 @@ -26,9 +25,9 @@ def write_bytes(filename, bytes_content): filename = to_unicode(filename) dir_name = fs.path.dirname(filename) the_file_name = fs.path.basename(filename) - with open_fs(dir_name) as the_fs: + with fs.open_fs(dir_name) as the_fs: the_fs.writebytes(the_file_name, bytes_content) - + def is_dir(path): path = to_unicode(path) diff --git a/moban/utils.py b/moban/utils.py index 3e2d0a9a..9f8078ea 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -4,12 +4,9 @@ import errno import logging -from moban import constants, exceptions -from moban import file_system +from moban import constants, exceptions, file_system import fs -from fs import path as fs_path -from fs import open_fs log = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -73,14 +70,7 @@ def write_file_out(filename, content): if dest_folder: mkdir_p(dest_folder) - if PY2: - if isinstance(filename, unicode) is False: - filename = unicode(filename) - dir_name = fs_path.dirname(filename) - the_file_name = fs_path.basename(filename) - with open_fs(dir_name) as the_fs: - the_fs.writebytes(the_file_name, content) - #file_system.write_bytes(filename, content) + file_system.write_bytes(filename, content) def mkdir_p(path): From 0d14f3255451c8709cca2c943ba5f4cee7664169 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 25 Jul 2019 00:16:40 +0100 Subject: [PATCH 037/167] :shirt: fix last lint error --- tests/test_repo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_repo.py b/tests/test_repo.py index c466b646..803e7a82 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,5 +1,3 @@ -import os - from mock import patch from moban.repo import ( git_clone, From 989a2c12c2052042c25240e915904af24a04b1cc Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 25 Jul 2019 18:33:07 +0100 Subject: [PATCH 038/167] :hammer: code refactoring --- .moban.cd/moban.yml | 2 +- min_requirements.txt | 2 +- moban/data_loaders/manager.py | 6 ++---- moban/file_system.py | 4 ++++ moban/mobanfile/templates.py | 34 ++++++++++++++++++---------------- moban/plugins/template.py | 15 ++++++--------- moban/repo.py | 6 ++---- moban/utils.py | 12 +++++------- requirements.txt | 2 +- setup.py | 2 +- 10 files changed, 41 insertions(+), 44 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 4136ed87..f06eb5ac 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -28,7 +28,7 @@ dependencies: - crayons>= 0.1.0 - GitPython>=2.0.0 - git-url-parse>=1.2.2 - - fs>=2.0.12 + - fs>=2.0.25 description: Yet another jinja2 cli command for static text generation scm_host: github.com lint_command: make install_test format git-diff-check lint diff --git a/min_requirements.txt b/min_requirements.txt index 155bc850..d509e0d3 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -8,4 +8,4 @@ appdirs==1.2.0 crayons== 0.1.0 GitPython==2.0.0 git-url-parse==1.2.2 -fs==2.0.12 +fs==2.0.25 diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index 86dcd16e..4aef6288 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -1,15 +1,13 @@ -from moban import utils, constants +from moban import utils, constants, file_system from lml.plugin import PluginManager -from fs import path - class AnyDataLoader(PluginManager): def __init__(self): super(AnyDataLoader, self).__init__(constants.DATA_LOADER_EXTENSION) def get_data(self, file_name): - file_extension = path.splitext(file_name)[1] + file_extension = file_system.splitext(file_name)[1] file_type = file_extension if file_extension.startswith("."): file_type = file_type[1:] diff --git a/moban/file_system.py b/moban/file_system.py index 0de9a105..459bc129 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -7,6 +7,10 @@ PY2 = sys.version_info[0] == 2 +join = fs.path.join +splitext = fs.path.splitext + + @contextmanager def open_fs(path): path = to_unicode(path) diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index cb8a9a16..726595eb 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -1,12 +1,8 @@ import logging -from moban import reporter -from moban import file_system as moban_fs +from moban import reporter, file_system from moban.utils import find_file_in_template_dirs -import fs -import fs.path - log = logging.getLogger(__name__) @@ -33,7 +29,7 @@ def handle_template(template_file, output, template_dirs): reporter.report_error_message( "{0} cannot be found".format(template_file) ) - elif moban_fs.is_dir(template_file_on_disk): + elif file_system.is_dir(template_file_on_disk): for a_triple in _list_dir_files( template_file, template_file_on_disk, output ): @@ -44,28 +40,34 @@ def handle_template(template_file, output, template_dirs): def _list_dir_files(source, actual_source_path, dest): - for file_name in moban_fs.list_dir(actual_source_path): - if moban_fs.is_file(fs.path.join(actual_source_path, file_name)): + for file_name in file_system.list_dir(actual_source_path): + if file_system.is_file( + file_system.join(actual_source_path, file_name) + ): # please note jinja2 does NOT like windows path # hence the following statement looks like cross platform # src_file_under_dir = os.path.join(source, file_name) # but actually it breaks windows instead. src_file_under_dir = "%s/%s" % (source, file_name) - dest_file_under_dir = fs.path.join(dest, file_name) + dest_file_under_dir = file_system.join(dest, file_name) template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) def _listing_directory_files_recusively(source, actual_source_path, dest): - for file_name in moban_fs.list_dir(actual_source_path): - src_file_under_dir = fs.path.join(source, file_name) - dest_file_under_dir = fs.path.join(dest, file_name) - real_src_file = fs.path.join(actual_source_path, file_name) - if moban_fs.is_file(fs.path.join(actual_source_path, file_name)): + for file_name in file_system.list_dir(actual_source_path): + src_file_under_dir = file_system.join(source, file_name) + dest_file_under_dir = file_system.join(dest, file_name) + real_src_file = file_system.join(actual_source_path, file_name) + if file_system.is_file( + file_system.join(actual_source_path, file_name) + ): template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) - elif moban_fs.is_dir(fs.path.join(actual_source_path, file_name)): + elif file_system.is_dir( + file_system.join(actual_source_path, file_name) + ): for a_triple in _listing_directory_files_recusively( src_file_under_dir, real_src_file, dest_file_under_dir ): @@ -73,7 +75,7 @@ def _listing_directory_files_recusively(source, actual_source_path, dest): def _get_template_type(template_file): - _, extension = fs.path.splitext(template_file) + _, extension = file_system.splitext(template_file) if extension: template_type = extension[1:] else: diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 64f44ae9..e0db8485 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -1,16 +1,13 @@ import sys import logging -from moban import repo, utils, reporter, constants, exceptions -from moban import file_system as moban_fs +from moban import repo, utils, reporter, constants, exceptions, file_system from lml.plugin import PluginManager from moban.hashstore import HASH_STORE from moban.plugins.context import Context from moban.plugins.library import LIBRARIES from moban.plugins.strategy import Strategy -import fs.path - log = logging.getLogger(__name__) PY3_ABOVE = sys.version_info[0] > 2 @@ -185,13 +182,13 @@ def expand_template_directory(directory): translated_directory = None if ":" in directory and directory[1] != ":": library_or_repo_name, relative_path = directory.split(":") - potential_repo_path = fs.path.join( + potential_repo_path = file_system.join( repo.get_moban_home(), library_or_repo_name ) - if moban_fs.exists(potential_repo_path): + if file_system.exists(potential_repo_path): # expand repo template path if relative_path: - translated_directory = fs.path.join( + translated_directory = file_system.join( potential_repo_path, relative_path ) else: @@ -200,12 +197,12 @@ def expand_template_directory(directory): # expand pypi template path library_path = LIBRARIES.resource_path_of(library_or_repo_name) if relative_path: - translated_directory = fs.path.join( + translated_directory = file_system.join( library_path, relative_path ) else: translated_directory = library_path else: # local template path - translated_directory = moban_fs.abspath(directory) + translated_directory = file_system.abspath(directory) return translated_directory diff --git a/moban/repo.py b/moban/repo.py index 1e4f3500..f23af9ad 100644 --- a/moban/repo.py +++ b/moban/repo.py @@ -4,8 +4,6 @@ from moban import reporter, constants, exceptions, file_system from moban.utils import mkdir_p -import fs.path - def git_clone(requires): from git import Repo @@ -21,7 +19,7 @@ def git_clone(requires): for require in requires: repo_name = get_repo_name(require.git_url) - local_repo_folder = fs.path.join(moban_home, repo_name) + local_repo_folder = file_system.join(moban_home, repo_name) if file_system.exists(local_repo_folder): reporter.report_git_pull(repo_name) repo = Repo(local_repo_folder) @@ -61,7 +59,7 @@ def get_moban_home(): from appdirs import user_cache_dir home_dir = user_cache_dir(appname=constants.PROGRAM_NAME) - return fs.path.join(home_dir, constants.MOBAN_REPOS_DIR_NAME) + return file_system.join(home_dir, constants.MOBAN_REPOS_DIR_NAME) def make_sure_git_is_available(): diff --git a/moban/utils.py b/moban/utils.py index 9f8078ea..1f6e2817 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -6,8 +6,6 @@ from moban import constants, exceptions, file_system -import fs - log = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -36,7 +34,7 @@ def search_file(base_dir, file_name): the_file = file_name if not file_system.exists(the_file): if base_dir: - the_file = fs.path.join(base_dir, file_name) + the_file = file_system.join(base_dir, file_name) if not file_system.exists(the_file): raise IOError( constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) @@ -94,11 +92,11 @@ def pip_install(packages): def get_template_path(template_dirs, template): for a_dir in template_dirs: template_file_exists = file_system.exists( - fs.path.join(a_dir, template) - ) and file_system.is_file(fs.path.join(a_dir, template)) + file_system.join(a_dir, template) + ) and file_system.is_file(file_system.join(a_dir, template)) if template_file_exists: - return fs.path.join(a_dir, template) + return file_system.join(a_dir, template) raise exceptions.FileNotFound @@ -125,7 +123,7 @@ def verify_the_existence_of_directories(dirs): def find_file_in_template_dirs(src, template_dirs): log.debug(template_dirs) for folder in template_dirs: - path = fs.path.join(folder, src) + path = file_system.join(folder, src) if file_system.exists(path): return path else: diff --git a/requirements.txt b/requirements.txt index 27e772e4..a02fc1b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ appdirs>=1.2.0 crayons>= 0.1.0 GitPython>=2.0.0 git-url-parse>=1.2.2 -fs>=2.0.12 +fs>=2.0.25 diff --git a/setup.py b/setup.py index f1a97b21..046ef3d9 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ "crayons>= 0.1.0", "GitPython>=2.0.0", "git-url-parse>=1.2.2", - "fs>=2.0.12", + "fs>=2.0.25", ] SETUP_COMMANDS = {} From 8ab0258e4ef07a5c6913b62d2f0d389809732185 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 25 Jul 2019 18:34:33 +0100 Subject: [PATCH 039/167] :books: update change log --- .moban.cd/changelog.yml | 6 ++++++ CHANGELOG.rst | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index bc1ba839..d11ca458 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,12 @@ name: moban organisation: moremoban releases: + - changes: + - action: Updated + details: + - "`#205`: support python file system 2" + date: tba + version: 0.6.0 - changes: - action: Updated details: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 21e34261..e0262c9f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.6.0 - tba +-------------------------------------------------------------------------------- + +Updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#205 `_: support python file + system 2 + 0.5.0 - 14.07.2019 -------------------------------------------------------------------------------- From 2281a4fa6ac8344ec5b782a7572b0b705c024b53 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 25 Jul 2019 18:42:12 +0100 Subject: [PATCH 040/167] :hammer: code refactoring --- moban/constants.py | 11 ++++++++--- moban/data_loaders/manager.py | 2 +- moban/file_system.py | 4 ++-- moban/mobanfile/templates.py | 16 ++++++++-------- moban/plugins/template.py | 6 +++--- moban/repo.py | 4 ++-- moban/utils.py | 10 +++++----- 7 files changed, 29 insertions(+), 24 deletions(-) diff --git a/moban/constants.py b/moban/constants.py index df476b0d..e7e3cee5 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -1,4 +1,4 @@ -import fs.path +from moban import file_system # Template type TEMPLATE_JINJA2 = "jinja2" @@ -41,9 +41,14 @@ DEFAULT_TEMPLATE_DIRNAME = ".%s.td" % PROGRAM_NAME DEFAULT_OPTIONS = { # .moban.cd, default configuration dir - LABEL_CONFIG_DIR: fs.path.join(".", DEFAULT_CONFIGURATION_DIRNAME), + LABEL_CONFIG_DIR: file_system.path_join( + ".", DEFAULT_CONFIGURATION_DIRNAME + ), # .moban.td, default template dirs - LABEL_TMPL_DIRS: [".", fs.path.join(".", DEFAULT_TEMPLATE_DIRNAME)], + LABEL_TMPL_DIRS: [ + ".", + file_system.path_join(".", DEFAULT_TEMPLATE_DIRNAME), + ], # moban.output, default output file name LABEL_OUTPUT: "%s.output" % PROGRAM_NAME, # data.yml, default data input file diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index 4aef6288..5ab9e9d3 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -7,7 +7,7 @@ def __init__(self): super(AnyDataLoader, self).__init__(constants.DATA_LOADER_EXTENSION) def get_data(self, file_name): - file_extension = file_system.splitext(file_name)[1] + file_extension = file_system.path_splitext(file_name)[1] file_type = file_extension if file_extension.startswith("."): file_type = file_type[1:] diff --git a/moban/file_system.py b/moban/file_system.py index 459bc129..bc2449ef 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -7,8 +7,8 @@ PY2 = sys.version_info[0] == 2 -join = fs.path.join -splitext = fs.path.splitext +path_join = fs.path.join +path_splitext = fs.path.splitext @contextmanager diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 726595eb..1c5f8538 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -42,7 +42,7 @@ def handle_template(template_file, output, template_dirs): def _list_dir_files(source, actual_source_path, dest): for file_name in file_system.list_dir(actual_source_path): if file_system.is_file( - file_system.join(actual_source_path, file_name) + file_system.path_join(actual_source_path, file_name) ): # please note jinja2 does NOT like windows path # hence the following statement looks like cross platform @@ -50,23 +50,23 @@ def _list_dir_files(source, actual_source_path, dest): # but actually it breaks windows instead. src_file_under_dir = "%s/%s" % (source, file_name) - dest_file_under_dir = file_system.join(dest, file_name) + dest_file_under_dir = file_system.path_join(dest, file_name) template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) def _listing_directory_files_recusively(source, actual_source_path, dest): for file_name in file_system.list_dir(actual_source_path): - src_file_under_dir = file_system.join(source, file_name) - dest_file_under_dir = file_system.join(dest, file_name) - real_src_file = file_system.join(actual_source_path, file_name) + src_file_under_dir = file_system.path_join(source, file_name) + dest_file_under_dir = file_system.path_join(dest, file_name) + real_src_file = file_system.path_join(actual_source_path, file_name) if file_system.is_file( - file_system.join(actual_source_path, file_name) + file_system.path_join(actual_source_path, file_name) ): template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) elif file_system.is_dir( - file_system.join(actual_source_path, file_name) + file_system.path_join(actual_source_path, file_name) ): for a_triple in _listing_directory_files_recusively( src_file_under_dir, real_src_file, dest_file_under_dir @@ -75,7 +75,7 @@ def _listing_directory_files_recusively(source, actual_source_path, dest): def _get_template_type(template_file): - _, extension = file_system.splitext(template_file) + _, extension = file_system.path_splitext(template_file) if extension: template_type = extension[1:] else: diff --git a/moban/plugins/template.py b/moban/plugins/template.py index e0db8485..70b6310d 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -182,13 +182,13 @@ def expand_template_directory(directory): translated_directory = None if ":" in directory and directory[1] != ":": library_or_repo_name, relative_path = directory.split(":") - potential_repo_path = file_system.join( + potential_repo_path = file_system.path_join( repo.get_moban_home(), library_or_repo_name ) if file_system.exists(potential_repo_path): # expand repo template path if relative_path: - translated_directory = file_system.join( + translated_directory = file_system.path_join( potential_repo_path, relative_path ) else: @@ -197,7 +197,7 @@ def expand_template_directory(directory): # expand pypi template path library_path = LIBRARIES.resource_path_of(library_or_repo_name) if relative_path: - translated_directory = file_system.join( + translated_directory = file_system.path_join( library_path, relative_path ) else: diff --git a/moban/repo.py b/moban/repo.py index f23af9ad..47c40635 100644 --- a/moban/repo.py +++ b/moban/repo.py @@ -19,7 +19,7 @@ def git_clone(requires): for require in requires: repo_name = get_repo_name(require.git_url) - local_repo_folder = file_system.join(moban_home, repo_name) + local_repo_folder = file_system.path_join(moban_home, repo_name) if file_system.exists(local_repo_folder): reporter.report_git_pull(repo_name) repo = Repo(local_repo_folder) @@ -59,7 +59,7 @@ def get_moban_home(): from appdirs import user_cache_dir home_dir = user_cache_dir(appname=constants.PROGRAM_NAME) - return file_system.join(home_dir, constants.MOBAN_REPOS_DIR_NAME) + return file_system.path_join(home_dir, constants.MOBAN_REPOS_DIR_NAME) def make_sure_git_is_available(): diff --git a/moban/utils.py b/moban/utils.py index 1f6e2817..94cbc9be 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -34,7 +34,7 @@ def search_file(base_dir, file_name): the_file = file_name if not file_system.exists(the_file): if base_dir: - the_file = file_system.join(base_dir, file_name) + the_file = file_system.path_join(base_dir, file_name) if not file_system.exists(the_file): raise IOError( constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) @@ -92,11 +92,11 @@ def pip_install(packages): def get_template_path(template_dirs, template): for a_dir in template_dirs: template_file_exists = file_system.exists( - file_system.join(a_dir, template) - ) and file_system.is_file(file_system.join(a_dir, template)) + file_system.path_join(a_dir, template) + ) and file_system.is_file(file_system.path_join(a_dir, template)) if template_file_exists: - return file_system.join(a_dir, template) + return file_system.path_join(a_dir, template) raise exceptions.FileNotFound @@ -123,7 +123,7 @@ def verify_the_existence_of_directories(dirs): def find_file_in_template_dirs(src, template_dirs): log.debug(template_dirs) for folder in template_dirs: - path = file_system.join(folder, src) + path = file_system.path_join(folder, src) if file_system.exists(path): return path else: From 438d6dee7a50fde516a7a279a65dd1a7edc67691 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 25 Jul 2019 18:48:03 +0100 Subject: [PATCH 041/167] :shirt: pin up python file system to 2.2.0 --- .moban.cd/moban.yml | 2 +- min_requirements.txt | 2 +- requirements.txt | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index f06eb5ac..5b94a155 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -28,7 +28,7 @@ dependencies: - crayons>= 0.1.0 - GitPython>=2.0.0 - git-url-parse>=1.2.2 - - fs>=2.0.25 + - fs>=2.2.0 description: Yet another jinja2 cli command for static text generation scm_host: github.com lint_command: make install_test format git-diff-check lint diff --git a/min_requirements.txt b/min_requirements.txt index d509e0d3..77d91697 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -8,4 +8,4 @@ appdirs==1.2.0 crayons== 0.1.0 GitPython==2.0.0 git-url-parse==1.2.2 -fs==2.0.25 +fs==2.2.0 diff --git a/requirements.txt b/requirements.txt index a02fc1b1..08c13c2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ appdirs>=1.2.0 crayons>= 0.1.0 GitPython>=2.0.0 git-url-parse>=1.2.2 -fs>=2.0.25 +fs>=2.2.0 diff --git a/setup.py b/setup.py index 046ef3d9..0e165f82 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ "crayons>= 0.1.0", "GitPython>=2.0.0", "git-url-parse>=1.2.2", - "fs>=2.0.25", + "fs>=2.2.0", ] SETUP_COMMANDS = {} From 400035a5036cd5764c9bc7b2217e31efae7f1fc8 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 25 Jul 2019 19:00:09 +0100 Subject: [PATCH 042/167] :fire: minor tuning --- moban/mobanfile/templates.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 1c5f8538..85a4e66e 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -60,14 +60,10 @@ def _listing_directory_files_recusively(source, actual_source_path, dest): src_file_under_dir = file_system.path_join(source, file_name) dest_file_under_dir = file_system.path_join(dest, file_name) real_src_file = file_system.path_join(actual_source_path, file_name) - if file_system.is_file( - file_system.path_join(actual_source_path, file_name) - ): + if file_system.is_file(real_src_file): template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) - elif file_system.is_dir( - file_system.path_join(actual_source_path, file_name) - ): + elif file_system.is_dir(real_src_file): for a_triple in _listing_directory_files_recusively( src_file_under_dir, real_src_file, dest_file_under_dir ): From dfd660e2d2cd9071f7f14bb892c1f1e96489741c Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 08:30:30 +0100 Subject: [PATCH 043/167] :lipstick: pump up version number --- .moban.cd/moban.yml | 4 ++-- docs/conf.py | 2 +- moban/_version.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 5b94a155..4aef13e7 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -4,8 +4,8 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.5.0 -current_version: 0.5.0 +version: 0.5.1 +current_version: 0.5.1 release: 0.5.0 branch: master master: index diff --git a/docs/conf.py b/docs/conf.py index 4f768e04..ee4da812 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,7 +25,7 @@ copyright = '2017-2019 Onni Software Ltd.' author = 'C. W.' # The short X.Y version -version = '0.5.0' +version = '0.5.1' # The full version, including alpha/beta/rc tags release = '0.5.0' diff --git a/moban/_version.py b/moban/_version.py index b407eb3a..9d1705a2 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.5.0" +__version__ = "0.5.1" __author__ = "C. W." diff --git a/setup.py b/setup.py index 0e165f82..1fed8525 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ NAME = "moban" AUTHOR = "C. W." -VERSION = "0.5.0" +VERSION = "0.5.1" EMAIL = "wangc_2011@hotmail.com" LICENSE = "MIT" ENTRY_POINTS = { From b3d427c35898fdfb94772c1f98ea3dbe3b5e78aa Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 18:55:29 +0100 Subject: [PATCH 044/167] :sparkles: python file system compliant file openers --- .moban.cd/moban.yml | 12 ++++++++---- docs/conf.py | 2 +- moban/_version.py | 2 +- moban/fs_openers.py | 31 +++++++++++++++++++++++++++++++ setup.py | 8 ++++++-- 5 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 moban/fs_openers.py diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 4aef13e7..c3e4336d 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -4,13 +4,17 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.5.1 -current_version: 0.5.1 +version: 0.6.0 +current_version: 0.6.0 release: 0.5.0 branch: master master: index -command_line_interface: "moban" -entry_point: "moban.main:main" +entry_points: + console_scripts: + - "moban = moban.main:main" + fs.opener: + - "pypi = moban.fs_openers:PypiFSOpener" + - "repo = moban.fs_openers:RepoFSOpener" company: Onni Software Ltd. copyright_year: 2017-2019 copyright: 2017-2019 Onni Software Ltd. and its contributors diff --git a/docs/conf.py b/docs/conf.py index ee4da812..8f4f4e27 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,7 +25,7 @@ copyright = '2017-2019 Onni Software Ltd.' author = 'C. W.' # The short X.Y version -version = '0.5.1' +version = '0.6.0' # The full version, including alpha/beta/rc tags release = '0.5.0' diff --git a/moban/_version.py b/moban/_version.py index 9d1705a2..6197ada2 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.5.1" +__version__ = "0.6.0" __author__ = "C. W." diff --git a/moban/fs_openers.py b/moban/fs_openers.py new file mode 100644 index 00000000..7c35d9ec --- /dev/null +++ b/moban/fs_openers.py @@ -0,0 +1,31 @@ +import fs.path +from fs.osfs import OSFS +from moban.plugins.library import LIBRARIES +from fs.opener import Opener +from moban import repo + + +class PypiFSOpener(Opener): + protocols = ['pypi'] + + def open_fs(self, fs_url, parse_result, writeable, create, cwd): + pypi_package_name, _, dir_path = parse_result.resource.partition('/') + library_path = LIBRARIES.resource_path_of(pypi_package_name) + root_path = fs.path.join(library_path, dir_path) + osfs = OSFS( + root_path=root_path + ) + return osfs + + +class RepoOpener(Opener): + protocols = ['repo'] + + def open_fs(self, fs_url, parse_result, writeable, create, cwd): + repo_name, _, dir_path = parse_result.resource.partition('/') + actual_repo_path = fs.path.join(repo.get_moban_home(), repo_name) + root_path = fs.path.join(actual_repo_path, dir_path) + osfs = OSFS( + root_path=root_path + ) + return osfs diff --git a/setup.py b/setup.py index 1fed8525..8c241c17 100644 --- a/setup.py +++ b/setup.py @@ -38,12 +38,16 @@ NAME = "moban" AUTHOR = "C. W." -VERSION = "0.5.1" +VERSION = "0.6.0" EMAIL = "wangc_2011@hotmail.com" LICENSE = "MIT" ENTRY_POINTS = { "console_scripts": [ - "moban = moban.main:main" + "moban = moban.main:main", + ], + "fs.opener": [ + "pypi = moban.fs_openers:PypiFSOpener", + "repo = moban.fs_openers:RepoFSOpener", ], } DESCRIPTION = ( From bc41a9564e3839797a99ee815f2b31ae1b5a7f64 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 18:58:19 +0100 Subject: [PATCH 045/167] :tractor: relocate level 9 docs to deprecated but kept for unit tests --- .../.moban.yml | 0 .../README.rst | 0 .../config.yml | 0 .../local/demo.txt.jj2 | 0 tests/test_docs.py | 4 ++-- 5 files changed, 2 insertions(+), 2 deletions(-) rename docs/{level-9-moban-dependency-as-pypi-package => deprecated-level-9-moban-dependency-as-pypi-package}/.moban.yml (100%) rename docs/{level-9-moban-dependency-as-pypi-package => deprecated-level-9-moban-dependency-as-pypi-package}/README.rst (100%) rename docs/{level-9-moban-dependency-as-pypi-package => deprecated-level-9-moban-dependency-as-pypi-package}/config.yml (100%) rename docs/{level-9-moban-dependency-as-pypi-package => deprecated-level-9-moban-dependency-as-pypi-package}/local/demo.txt.jj2 (100%) diff --git a/docs/level-9-moban-dependency-as-pypi-package/.moban.yml b/docs/deprecated-level-9-moban-dependency-as-pypi-package/.moban.yml similarity index 100% rename from docs/level-9-moban-dependency-as-pypi-package/.moban.yml rename to docs/deprecated-level-9-moban-dependency-as-pypi-package/.moban.yml diff --git a/docs/level-9-moban-dependency-as-pypi-package/README.rst b/docs/deprecated-level-9-moban-dependency-as-pypi-package/README.rst similarity index 100% rename from docs/level-9-moban-dependency-as-pypi-package/README.rst rename to docs/deprecated-level-9-moban-dependency-as-pypi-package/README.rst diff --git a/docs/level-9-moban-dependency-as-pypi-package/config.yml b/docs/deprecated-level-9-moban-dependency-as-pypi-package/config.yml similarity index 100% rename from docs/level-9-moban-dependency-as-pypi-package/config.yml rename to docs/deprecated-level-9-moban-dependency-as-pypi-package/config.yml diff --git a/docs/level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 b/docs/deprecated-level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 similarity index 100% rename from docs/level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 rename to docs/deprecated-level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 diff --git a/tests/test_docs.py b/tests/test_docs.py index 7f78cb30..805a7be1 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -117,9 +117,9 @@ def test_level_8(self): check_file = os.path.join("templated-folder", "my") self._raw_moban(["moban"], folder, expected, check_file) - def test_level_9(self): + def test_level_9_deprecated(self): expected = "pypi-mobans: moban dependency as pypi package" - folder = "level-9-moban-dependency-as-pypi-package" + folder = "deprecated-level-9-moban-dependency-as-pypi-package" self._raw_moban(["moban"], folder, expected, "test.txt") def test_level_10(self): From 9a0bc5f3f9e805acb1806138a3611091ecc22b0c Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 19:06:45 +0100 Subject: [PATCH 046/167] :microscope: regression test new pypi syntax --- .gitignore | 1 + .moban.d/moban_gitignore.jj2 | 1 + .../.moban.yml | 11 ++++ .../README.rst | 50 +++++++++++++++++++ .../config.yml | 2 + moban/plugins/template.py | 2 +- tests/test_docs.py | 5 ++ 7 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 docs/level-9-moban-dependency-as-pypi-package/.moban.yml create mode 100644 docs/level-9-moban-dependency-as-pypi-package/README.rst create mode 100644 docs/level-9-moban-dependency-as-pypi-package/config.yml diff --git a/.gitignore b/.gitignore index 5ef0acc0..6940c0b9 100644 --- a/.gitignore +++ b/.gitignore @@ -506,6 +506,7 @@ cscope.in.out cscope.po.out +docs/deprecated-level-9-moban-dependency-as-pypi-package/mytravis.yml docs/level-1-jinja2-cli/testout docs/level-10-moban-dependency-as-git-repo/mytravis.yml docs/level-18-user-defined-template-types/b.output diff --git a/.moban.d/moban_gitignore.jj2 b/.moban.d/moban_gitignore.jj2 index 3ed4a68e..02abd612 100644 --- a/.moban.d/moban_gitignore.jj2 +++ b/.moban.d/moban_gitignore.jj2 @@ -1,6 +1,7 @@ {% extends "gitignore.jj2" %} {% block extra %} +docs/deprecated-level-9-moban-dependency-as-pypi-package/mytravis.yml docs/level-1-jinja2-cli/testout docs/level-10-moban-dependency-as-git-repo/mytravis.yml docs/level-18-user-defined-template-types/b.output diff --git a/docs/level-9-moban-dependency-as-pypi-package/.moban.yml b/docs/level-9-moban-dependency-as-pypi-package/.moban.yml new file mode 100644 index 00000000..f9e7bc46 --- /dev/null +++ b/docs/level-9-moban-dependency-as-pypi-package/.moban.yml @@ -0,0 +1,11 @@ +requires: + - pypi-mobans-pkg +configuration: + template_dir: + - "pypi://pypi-mobans-pkg/templates" + - local + configuration: config.yml + configuration_dir: "pypi://pypi-mobans-pkg/config" +targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 diff --git a/docs/level-9-moban-dependency-as-pypi-package/README.rst b/docs/level-9-moban-dependency-as-pypi-package/README.rst new file mode 100644 index 00000000..cd096b67 --- /dev/null +++ b/docs/level-9-moban-dependency-as-pypi-package/README.rst @@ -0,0 +1,50 @@ +level 9: moban dependency as pypi package +================================================================================ + +Why not enable template reuse? Once a template is written somewhere by somebody, +as long as it is good and useful, it is always to reuse it, isn't it? DRY +principle kicks in. + +Now with moban, it is possible to package up your mobans/templates +into a pypi package and distribute it to the world of moban. + + +Here are the sample file:: + + requires: + - pypi-mobans-pkg + configuration: + template_dir: + - "pypi://pypi-mobans-pkg/templates" + configuration: config.yml + configuration_dir: "pypi://pypi-mobans-pkg/config" + targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 + +where `requires` lead to a list of pypi packages. The short syntax is:: + + requires: + - python-package-name + +When you refer to it in configuration section, here is the syntax:: + + configuration: + - template_dir: + - "python-package-name:relative-folder-inside-the-package" + +Note: when you do not have relative directory, please keep semi-colon:: + + configuration: + template_dir: + - "python-package-name:" + +Alternative syntax +-------------------------------------------------------------------------------- + +The alternative syntax is:: + + requires: + - type: pypi + name: pypi-mobans + ... diff --git a/docs/level-9-moban-dependency-as-pypi-package/config.yml b/docs/level-9-moban-dependency-as-pypi-package/config.yml new file mode 100644 index 00000000..97730934 --- /dev/null +++ b/docs/level-9-moban-dependency-as-pypi-package/config.yml @@ -0,0 +1,2 @@ +overrides: data.yml +level9: "moban dependency as pypi package" diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 70b6310d..f43eab13 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -180,7 +180,7 @@ def expand_template_directory(directory): log.debug("Expanding %s..." % directory) translated_directory = None - if ":" in directory and directory[1] != ":": + if ":" in directory and "://" not in directory: library_or_repo_name, relative_path = directory.split(":") potential_repo_path = file_system.path_join( repo.get_moban_home(), library_or_repo_name diff --git a/tests/test_docs.py b/tests/test_docs.py index 805a7be1..0b337b7a 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -117,6 +117,11 @@ def test_level_8(self): check_file = os.path.join("templated-folder", "my") self._raw_moban(["moban"], folder, expected, check_file) + def test_level_9(self): + expected = "pypi-mobans: moban dependency as pypi package" + folder = "level-9-moban-dependency-as-pypi-package" + self._raw_moban(["moban"], folder, expected, "test.txt") + def test_level_9_deprecated(self): expected = "pypi-mobans: moban dependency as pypi package" folder = "deprecated-level-9-moban-dependency-as-pypi-package" From 41e177279cba3c34dc16c62fdfdef1414b6c914d Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 19:07:00 +0100 Subject: [PATCH 047/167] :lipstick: update coding style --- moban/fs_openers.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/moban/fs_openers.py b/moban/fs_openers.py index 7c35d9ec..97b02c89 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -1,31 +1,28 @@ +from moban import repo +from moban.plugins.library import LIBRARIES + import fs.path from fs.osfs import OSFS -from moban.plugins.library import LIBRARIES from fs.opener import Opener -from moban import repo class PypiFSOpener(Opener): - protocols = ['pypi'] + protocols = ["pypi"] def open_fs(self, fs_url, parse_result, writeable, create, cwd): - pypi_package_name, _, dir_path = parse_result.resource.partition('/') + pypi_package_name, _, dir_path = parse_result.resource.partition("/") library_path = LIBRARIES.resource_path_of(pypi_package_name) root_path = fs.path.join(library_path, dir_path) - osfs = OSFS( - root_path=root_path - ) + osfs = OSFS(root_path=root_path) return osfs class RepoOpener(Opener): - protocols = ['repo'] + protocols = ["repo"] def open_fs(self, fs_url, parse_result, writeable, create, cwd): - repo_name, _, dir_path = parse_result.resource.partition('/') + repo_name, _, dir_path = parse_result.resource.partition("/") actual_repo_path = fs.path.join(repo.get_moban_home(), repo_name) root_path = fs.path.join(actual_repo_path, dir_path) - osfs = OSFS( - root_path=root_path - ) + osfs = OSFS(root_path=root_path) return osfs From 45c3b9c0a7cc9662498d48d67497423d3dc7b9b9 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 19:10:57 +0100 Subject: [PATCH 048/167] :tractor: deprecate old level 10 git repo syntax --- .gitignore | 1 + .moban.d/moban_gitignore.jj2 | 1 + .../.moban.yml | 0 .../README.rst | 0 .../config.yml | 0 .../local/demo.txt.jj2 | 0 .../local/mytravis.yml | 0 tests/test_docs.py | 4 ++-- 8 files changed, 4 insertions(+), 2 deletions(-) rename docs/{level-10-moban-dependency-as-git-repo => deprecated-level-10-moban-dependency-as-git-repo}/.moban.yml (100%) rename docs/{level-10-moban-dependency-as-git-repo => deprecated-level-10-moban-dependency-as-git-repo}/README.rst (100%) rename docs/{level-10-moban-dependency-as-git-repo => deprecated-level-10-moban-dependency-as-git-repo}/config.yml (100%) rename docs/{level-10-moban-dependency-as-git-repo => deprecated-level-10-moban-dependency-as-git-repo}/local/demo.txt.jj2 (100%) rename docs/{level-10-moban-dependency-as-git-repo => deprecated-level-10-moban-dependency-as-git-repo}/local/mytravis.yml (100%) diff --git a/.gitignore b/.gitignore index 6940c0b9..95e2ceaf 100644 --- a/.gitignore +++ b/.gitignore @@ -507,6 +507,7 @@ cscope.po.out docs/deprecated-level-9-moban-dependency-as-pypi-package/mytravis.yml +docs/deprecated-level-10-moban-dependency-as-git-repo/mytravis.yml docs/level-1-jinja2-cli/testout docs/level-10-moban-dependency-as-git-repo/mytravis.yml docs/level-18-user-defined-template-types/b.output diff --git a/.moban.d/moban_gitignore.jj2 b/.moban.d/moban_gitignore.jj2 index 02abd612..7f19ac2f 100644 --- a/.moban.d/moban_gitignore.jj2 +++ b/.moban.d/moban_gitignore.jj2 @@ -2,6 +2,7 @@ {% block extra %} docs/deprecated-level-9-moban-dependency-as-pypi-package/mytravis.yml +docs/deprecated-level-10-moban-dependency-as-git-repo/mytravis.yml docs/level-1-jinja2-cli/testout docs/level-10-moban-dependency-as-git-repo/mytravis.yml docs/level-18-user-defined-template-types/b.output diff --git a/docs/level-10-moban-dependency-as-git-repo/.moban.yml b/docs/deprecated-level-10-moban-dependency-as-git-repo/.moban.yml similarity index 100% rename from docs/level-10-moban-dependency-as-git-repo/.moban.yml rename to docs/deprecated-level-10-moban-dependency-as-git-repo/.moban.yml diff --git a/docs/level-10-moban-dependency-as-git-repo/README.rst b/docs/deprecated-level-10-moban-dependency-as-git-repo/README.rst similarity index 100% rename from docs/level-10-moban-dependency-as-git-repo/README.rst rename to docs/deprecated-level-10-moban-dependency-as-git-repo/README.rst diff --git a/docs/level-10-moban-dependency-as-git-repo/config.yml b/docs/deprecated-level-10-moban-dependency-as-git-repo/config.yml similarity index 100% rename from docs/level-10-moban-dependency-as-git-repo/config.yml rename to docs/deprecated-level-10-moban-dependency-as-git-repo/config.yml diff --git a/docs/level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 b/docs/deprecated-level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 similarity index 100% rename from docs/level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 rename to docs/deprecated-level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 diff --git a/docs/level-10-moban-dependency-as-git-repo/local/mytravis.yml b/docs/deprecated-level-10-moban-dependency-as-git-repo/local/mytravis.yml similarity index 100% rename from docs/level-10-moban-dependency-as-git-repo/local/mytravis.yml rename to docs/deprecated-level-10-moban-dependency-as-git-repo/local/mytravis.yml diff --git a/tests/test_docs.py b/tests/test_docs.py index 0b337b7a..f2fd7a3a 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -127,9 +127,9 @@ def test_level_9_deprecated(self): folder = "deprecated-level-9-moban-dependency-as-pypi-package" self._raw_moban(["moban"], folder, expected, "test.txt") - def test_level_10(self): + def test_level_10_deprecated(self): expected = "pypi-mobans: moban dependency as git repo" - folder = "level-10-moban-dependency-as-git-repo" + folder = "deprecated-level-10-moban-dependency-as-git-repo" self._raw_moban(["moban"], folder, expected, "test.txt") def test_level_11(self): From 3bb05695ebec98f9cd3713fdd463c34b733835f2 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 19:16:41 +0100 Subject: [PATCH 049/167] :microscope: regression test repo://repo_name/path --- .../.moban.yml | 11 +++++ .../README.rst | 42 +++++++++++++++++++ .../config.yml | 2 + moban/fs_openers.py | 2 +- tests/test_docs.py | 5 +++ 5 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 docs/level-10-moban-dependency-as-git-repo/.moban.yml create mode 100644 docs/level-10-moban-dependency-as-git-repo/README.rst create mode 100644 docs/level-10-moban-dependency-as-git-repo/config.yml diff --git a/docs/level-10-moban-dependency-as-git-repo/.moban.yml b/docs/level-10-moban-dependency-as-git-repo/.moban.yml new file mode 100644 index 00000000..884a8206 --- /dev/null +++ b/docs/level-10-moban-dependency-as-git-repo/.moban.yml @@ -0,0 +1,11 @@ +requires: + - https://github.com/moremoban/pypi-mobans +configuration: + template_dir: + - "repo://pypi-mobans/templates" + - local + configuration: config.yml + configuration_dir: "repo://pypi-mobans/config" +targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 diff --git a/docs/level-10-moban-dependency-as-git-repo/README.rst b/docs/level-10-moban-dependency-as-git-repo/README.rst new file mode 100644 index 00000000..61de19db --- /dev/null +++ b/docs/level-10-moban-dependency-as-git-repo/README.rst @@ -0,0 +1,42 @@ +level 10: moban dependency as git repo +================================================================================ + +Since the support to have a pypi package as dependency, the moban pro user will +find it more useful to have git repo so that the changes to static content +could get propagate as it happens using git push and git pull. + +For now, github.com, gitlab.com and bitbucket.com are supported. Pull request +is welcome to add or improve this feature. + + +Here are the sample file:: + + requires: + - https://github.com/moremoban/pypi-mobans + configuration: + template_dir: + - "repo://pypi-mobans/templates" + - local + configuration: config.yml + configuration_dir: "repo://pypi-mobans/config" + targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 + +where `requires` lead to a list of pypi packages. And when you refer to it, +as in level-9 section, please use "pypi-mobans:" + + +Alternative syntax when submodule exists +-------------------------------------------------------------------------------- + +The alternative syntax is:: + + requires: + - type: git + url: https://github.com/your-git-url + submodule: true + branch: your_choice_or_default_branch_if_not_specified + reference: your_alternative_reference_but_not_used_together_with_branch + ... + diff --git a/docs/level-10-moban-dependency-as-git-repo/config.yml b/docs/level-10-moban-dependency-as-git-repo/config.yml new file mode 100644 index 00000000..a4d9f0c9 --- /dev/null +++ b/docs/level-10-moban-dependency-as-git-repo/config.yml @@ -0,0 +1,2 @@ +overrides: data.yml +level10: "moban dependency as git repo" diff --git a/moban/fs_openers.py b/moban/fs_openers.py index 97b02c89..7ab47651 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -17,7 +17,7 @@ def open_fs(self, fs_url, parse_result, writeable, create, cwd): return osfs -class RepoOpener(Opener): +class RepoFSOpener(Opener): protocols = ["repo"] def open_fs(self, fs_url, parse_result, writeable, create, cwd): diff --git a/tests/test_docs.py b/tests/test_docs.py index f2fd7a3a..ad7b4487 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -127,6 +127,11 @@ def test_level_9_deprecated(self): folder = "deprecated-level-9-moban-dependency-as-pypi-package" self._raw_moban(["moban"], folder, expected, "test.txt") + def test_level_10(self): + expected = "pypi-mobans: moban dependency as git repo" + folder = "level-10-moban-dependency-as-git-repo" + self._raw_moban(["moban"], folder, expected, "test.txt") + def test_level_10_deprecated(self): expected = "pypi-mobans: moban dependency as git repo" folder = "deprecated-level-10-moban-dependency-as-git-repo" From 5362983dbe292869096cfc0b2379f8dfa4f2b1a2 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 19:54:14 +0100 Subject: [PATCH 050/167] :green_heart: install the python file system entry points --- .azure-pipelines-steps-macos.yml | 1 + .azure-pipelines-steps.yml | 1 + .travis.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.azure-pipelines-steps-macos.yml b/.azure-pipelines-steps-macos.yml index 8958395b..0198ca11 100644 --- a/.azure-pipelines-steps-macos.yml +++ b/.azure-pipelines-steps-macos.yml @@ -6,6 +6,7 @@ steps: test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt pip install -r requirements.txt pip install -r tests/requirements.txt + python setup.py develop displayName: 'Setup dependencies' - script: | make diff --git a/.azure-pipelines-steps.yml b/.azure-pipelines-steps.yml index 2ef569c6..68783aa0 100644 --- a/.azure-pipelines-steps.yml +++ b/.azure-pipelines-steps.yml @@ -6,6 +6,7 @@ steps: if exist rnd_requirements.txt pip install -r rnd_requirements.txt pip install -r requirements.txt pip install -r tests\requirements.txt + python setup.py develop displayName: 'Setup dependencies' - script: | test.bat diff --git a/.travis.yml b/.travis.yml index 4f34a3e4..20a0e95c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,7 @@ before_install: pip install --no-deps -r rnd_requirements.txt - test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt ; - pip install -r tests/requirements.txt + - python setup.py develop script: - make test after_success: From 6a16f58bf0c9448585ce323c32eeb05f59f9cb3f Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 20:02:16 +0100 Subject: [PATCH 051/167] :newspaper: add missing files --- .../local/demo.txt.jj2 | 1 + .../local/mytravis.yml | 24 +++++++++++++++++++ .../local/demo.txt.jj2 | 1 + 3 files changed, 26 insertions(+) create mode 100644 docs/level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 create mode 100644 docs/level-10-moban-dependency-as-git-repo/local/mytravis.yml create mode 100644 docs/level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 diff --git a/docs/level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 b/docs/level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 new file mode 100644 index 00000000..a21dd010 --- /dev/null +++ b/docs/level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 @@ -0,0 +1 @@ +{{name}}: {{level10}} \ No newline at end of file diff --git a/docs/level-10-moban-dependency-as-git-repo/local/mytravis.yml b/docs/level-10-moban-dependency-as-git-repo/local/mytravis.yml new file mode 100644 index 00000000..dc7bd8c8 --- /dev/null +++ b/docs/level-10-moban-dependency-as-git-repo/local/mytravis.yml @@ -0,0 +1,24 @@ +sudo: false +language: python +notifications: + email: false +python: + - pypy-5.3.1 + - 3.7-dev + - 3.6 + - 3.5 + - 3.4 + - 2.7 +before_install: + - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi + - if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then + mv min_requirements.txt requirements.txt ; + fi + - test ! -f rnd_requirements.txt || + pip install --no-deps -r rnd_requirements.txt + - test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt ; + - pip install -r tests/requirements.txt +script: + - make test +after_success: + codecov diff --git a/docs/level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 b/docs/level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 new file mode 100644 index 00000000..ee50778b --- /dev/null +++ b/docs/level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 @@ -0,0 +1 @@ +{{name}}: {{level9}} \ No newline at end of file From 9511469f8c11911b67869121cb2a2f9fc757e976 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 20:09:37 +0100 Subject: [PATCH 052/167] :bug: fix windows bug c:/ , where c is found to be a pypi repo --- moban/plugins/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index f43eab13..5b705ad2 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -180,7 +180,7 @@ def expand_template_directory(directory): log.debug("Expanding %s..." % directory) translated_directory = None - if ":" in directory and "://" not in directory: + if ":" in directory and directory[1] != ":" and "://" not in directory: library_or_repo_name, relative_path = directory.split(":") potential_repo_path = file_system.path_join( repo.get_moban_home(), library_or_repo_name From 0632da3f88a4444cf68bd5f7aaa1357e50d9cdeb Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 20:15:05 +0100 Subject: [PATCH 053/167] :green_heart: make test --- .travis.yml | 1 - test.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20a0e95c..4f34a3e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,6 @@ before_install: pip install --no-deps -r rnd_requirements.txt - test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt ; - pip install -r tests/requirements.txt - - python setup.py develop script: - make test after_success: diff --git a/test.sh b/test.sh index b294f53a..a2a61ade 100644 --- a/test.sh +++ b/test.sh @@ -1,3 +1,3 @@ pip freeze - +python setup.py develop nosetests --with-cov --with-doctest --doctest-extension=.rst --cover-package moban --cover-package tests From f5ec738ac3e2230dbc1dd73395365d872ebe2a39 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 26 Jul 2019 20:28:17 +0100 Subject: [PATCH 054/167] :bug: min requirement for appdir has to be 1.4.3, fs 2.2.0 requires 1.4.3 --- .moban.cd/moban.yml | 2 +- min_requirements.txt | 2 +- requirements.txt | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index c3e4336d..63ada6fe 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -28,7 +28,7 @@ dependencies: - ruamel.yaml>=0.15.98;python_version == '3.8' - jinja2>=2.7.1 - lml>=0.0.9 - - appdirs>=1.2.0 + - appdirs>=1.4.3 - crayons>= 0.1.0 - GitPython>=2.0.0 - git-url-parse>=1.2.2 diff --git a/min_requirements.txt b/min_requirements.txt index 77d91697..48290b81 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -4,7 +4,7 @@ ruamel.yaml==0.15.5;python_version != '3.4' and python_version < '3.7' ruamel.yaml==0.15.98;python_version == '3.8' jinja2==2.7.1 lml==0.0.9 -appdirs==1.2.0 +appdirs==1.4.3 crayons== 0.1.0 GitPython==2.0.0 git-url-parse==1.2.2 diff --git a/requirements.txt b/requirements.txt index 08c13c2c..e368ffe7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ ruamel.yaml>=0.15.5;python_version != '3.4' and python_version < '3.7' ruamel.yaml>=0.15.98;python_version == '3.8' jinja2>=2.7.1 lml>=0.0.9 -appdirs>=1.2.0 +appdirs>=1.4.3 crayons>= 0.1.0 GitPython>=2.0.0 git-url-parse>=1.2.2 diff --git a/setup.py b/setup.py index 8c241c17..49706583 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ INSTALL_REQUIRES = [ "jinja2>=2.7.1", "lml>=0.0.9", - "appdirs>=1.2.0", + "appdirs>=1.4.3", "crayons>= 0.1.0", "GitPython>=2.0.0", "git-url-parse>=1.2.2", From 45ceaf543bfc0b414cf3d2d6e13480be05649a1c Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 27 Jul 2019 10:58:10 +0100 Subject: [PATCH 055/167] :fire: deprecated old syntax. and add read_unicode for moban-handlebars --- moban/constants.py | 4 ++++ moban/file_system.py | 9 +++++++ moban/plugins/template.py | 50 +++++++++++++++++++++++---------------- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/moban/constants.py b/moban/constants.py index e7e3cee5..93967b09 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -128,3 +128,7 @@ MESSAGE_DEPRECATE_COPY_SINCE_0_4_0 = ( "'%s:' is deprecated since 0.4.0! " + "Please use 'targets:' instead" ) % TEMPLATE_COPY +MESSAGE_DEPRECATE_MOBAN_NOTATION_SINCE_0_6_0 = ( + "File path notation is deprecated since 0.6.0! " + + "Please use pypi:// or repo:// notation instead" +) diff --git a/moban/file_system.py b/moban/file_system.py index bc2449ef..b550ea09 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -25,6 +25,15 @@ def open_fs(path): the_fs.close() +def read_unicode(path): + path = to_unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as fs_system: + with fs_system.open(the_file_name) as file_handle: + return file_handle.read() + + def write_bytes(filename, bytes_content): filename = to_unicode(filename) dir_name = fs.path.dirname(filename) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 5b705ad2..03ca26d6 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -4,6 +4,7 @@ from moban import repo, utils, reporter, constants, exceptions, file_system from lml.plugin import PluginManager from moban.hashstore import HASH_STORE +from moban.deprecated import deprecated from moban.plugins.context import Context from moban.plugins.library import LIBRARIES from moban.plugins.strategy import Strategy @@ -181,28 +182,35 @@ def expand_template_directory(directory): translated_directory = None if ":" in directory and directory[1] != ":" and "://" not in directory: - library_or_repo_name, relative_path = directory.split(":") - potential_repo_path = file_system.path_join( - repo.get_moban_home(), library_or_repo_name - ) - if file_system.exists(potential_repo_path): - # expand repo template path - if relative_path: - translated_directory = file_system.path_join( - potential_repo_path, relative_path - ) - else: - translated_directory = potential_repo_path - else: - # expand pypi template path - library_path = LIBRARIES.resource_path_of(library_or_repo_name) - if relative_path: - translated_directory = file_system.path_join( - library_path, relative_path - ) - else: - translated_directory = library_path + translated_directory = deprecated_moban_path_notation(directory) else: # local template path translated_directory = file_system.abspath(directory) return translated_directory + + +@deprecated(constants.MESSAGE_DEPRECATE_MOBAN_NOTATION_SINCE_0_6_0) +def deprecated_moban_path_notation(directory): + translated_directory = None + library_or_repo_name, relative_path = directory.split(":") + potential_repo_path = file_system.path_join( + repo.get_moban_home(), library_or_repo_name + ) + if file_system.exists(potential_repo_path): + # expand repo template path + if relative_path: + translated_directory = file_system.path_join( + potential_repo_path, relative_path + ) + else: + translated_directory = potential_repo_path + else: + # expand pypi template path + library_path = LIBRARIES.resource_path_of(library_or_repo_name) + if relative_path: + translated_directory = file_system.path_join( + library_path, relative_path + ) + else: + translated_directory = library_path + return translated_directory From 904324a4bdfa922e528eff3b01739019bddc1a9f Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 27 Jul 2019 11:58:27 +0100 Subject: [PATCH 056/167] :books: update change log --- .moban.cd/changelog.yml | 2 +- CHANGELOG.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index d11ca458..7d51f365 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -4,7 +4,7 @@ releases: - changes: - action: Updated details: - - "`#205`: support python file system 2" + - "`#205`: support `pyFilesystem2 `_" date: tba version: 0.6.0 - changes: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0262c9f..eee86ed8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,8 +7,8 @@ Change log Updated ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -#. `#205 `_: support python file - system 2 +#. `#205 `_: support + `pyFilesystem2 `_ 0.5.0 - 14.07.2019 -------------------------------------------------------------------------------- From d8ea6197283c63213fd47af044c769beaa2bc7b0 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 27 Jul 2019 11:59:17 +0100 Subject: [PATCH 057/167] :newspaper: address unicode related review comments --- lint.sh | 2 +- moban/hashstore.py | 5 ++--- moban/plugins/template.py | 2 +- moban/utils.py | 5 ++--- test.bat | 2 +- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lint.sh b/lint.sh index a0071dff..08f7c655 100644 --- a/lint.sh +++ b/lint.sh @@ -1 +1 @@ -flake8 --max-line-length=88 --exclude=.moban.d,docs --ignore=W503,W504 --builtins=unicode +flake8 --max-line-length=88 --exclude=.moban.d,docs --ignore=W503,W504 diff --git a/moban/hashstore.py b/moban/hashstore.py index 8e2b07d5..25b72518 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -65,9 +65,8 @@ def get_file_hash(afile): def get_hash(content): md5 = hashlib.md5() - if PY2: - if isinstance(content, unicode): - content = content.encode("utf-8") + if PY2 and content.__class__.__name__ == "unicode": + content = content.encode("utf-8") md5.update(content) return md5.digest().decode("latin1") diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 03ca26d6..34a85e48 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -185,7 +185,7 @@ def expand_template_directory(directory): translated_directory = deprecated_moban_path_notation(directory) else: # local template path - translated_directory = file_system.abspath(directory) + translated_directory = directory return translated_directory diff --git a/moban/utils.py b/moban/utils.py index 94cbc9be..93879e92 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -61,9 +61,8 @@ def file_permissions(afile): def write_file_out(filename, content): - if PY2: - if isinstance(content, unicode): - content = content.encode("utf-8") + if PY2 and content.__class__.__name__ == "unicode": + content = content.encode("utf-8") dest_folder = os.path.dirname(filename) if dest_folder: mkdir_p(dest_folder) diff --git a/test.bat b/test.bat index e0b1ba5f..399c6eda 100644 --- a/test.bat +++ b/test.bat @@ -2,7 +2,7 @@ pip freeze nosetests --with-coverage --cover-package=moban --cover-package=tests || goto :error -flake8 --max-line-length=88 --exclude=docs,.moban.d --ignore=W503,W504 --builtins=unicode|| goto :error +flake8 --max-line-length=88 --exclude=docs,.moban.d --ignore=W503,W504 || goto :error :error echo Failed with error #%errorlevel%. From 08ec831752a4a9f31e23671c200c7655195d31ec Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 27 Jul 2019 12:00:09 +0100 Subject: [PATCH 058/167] :hammer: use u for unicode convertion and :microscope: log errors from pyfilesystem2 --- moban/file_system.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index b550ea09..88fb9328 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -1,16 +1,32 @@ import sys +import logging from contextlib import contextmanager +from moban import reporter + import fs import fs.path PY2 = sys.version_info[0] == 2 - +LOG = logging.getLogger(__name__) path_join = fs.path.join path_splitext = fs.path.splitext +def log_fs_failure(function_in_this_module): + def wrapper(*args, **kwds): + try: + return function_in_this_module(*args, **kwds) + except fs.errors.CreateFailed: + message = "Failed to open %s" % args[0] + LOG.debug(message) + reporter.report_error_message(message) + raise + + return wrapper + + @contextmanager def open_fs(path): path = to_unicode(path) @@ -78,16 +94,17 @@ def list_dir(path): yield file_name +@log_fs_failure def abspath(path): path = to_unicode(path) dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as the_fs: return the_fs.getsyspath(the_file_name) def to_unicode(path): - if PY2: - if isinstance(path, unicode) is False: - return unicode(path) + if PY2 and path.__class__.__name__ != "unicode": + return u"".__class__(path) return path From 3b1e7819180412fb1a6cf42afd0c1dcedaa94c14 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 27 Jul 2019 19:20:00 +0100 Subject: [PATCH 059/167] :bug: return directory as it is if it is FSURL. need to assume all url as FSURL --- moban/plugins/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 34a85e48..03ca26d6 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -185,7 +185,7 @@ def expand_template_directory(directory): translated_directory = deprecated_moban_path_notation(directory) else: # local template path - translated_directory = directory + translated_directory = file_system.abspath(directory) return translated_directory From 8b24e8316e81cc3e55648ce69452432e54bcb06d Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 27 Jul 2019 19:43:52 +0100 Subject: [PATCH 060/167] :bug: avoid circular import --- moban/file_system.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index 88fb9328..fddfd0eb 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -2,8 +2,6 @@ import logging from contextlib import contextmanager -from moban import reporter - import fs import fs.path @@ -19,6 +17,7 @@ def wrapper(*args, **kwds): try: return function_in_this_module(*args, **kwds) except fs.errors.CreateFailed: + from moban import reporter message = "Failed to open %s" % args[0] LOG.debug(message) reporter.report_error_message(message) From 6c1d512f02414717d9567182df51906a83b82f9e Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 27 Jul 2019 19:55:20 +0100 Subject: [PATCH 061/167] :green_heart: make flake8 happy --- moban/file_system.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moban/file_system.py b/moban/file_system.py index fddfd0eb..fc0616bd 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -18,6 +18,7 @@ def wrapper(*args, **kwds): return function_in_this_module(*args, **kwds) except fs.errors.CreateFailed: from moban import reporter + message = "Failed to open %s" % args[0] LOG.debug(message) reporter.report_error_message(message) From 61b4835d35a582d20b4a487ac36880d1f1c58552 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 00:42:32 +0100 Subject: [PATCH 062/167] :sparkles: add templates in a zip demo. resolves #305 --- .../.moban.yml | 13 ++++ .../README.rst | 57 ++++++++++++++++++ .../a.output | 9 +++ .../cool-templates/base.jj2 | 11 ++++ .../cool-templates/cool.template.jj2 | 1 + .../custom-config/data.base.yaml | 2 + .../subfolder/template.in.zip.jj2 | 9 +++ .../custom-templates/template.in.zip.jj2 | 9 +++ .../data.yml | 2 + .../data2.yml | 2 + .../templates.zip | Bin 0 -> 510 bytes moban/copy/__init__.py | 5 +- moban/file_system.py | 38 ++++++++++++ moban/jinja2/engine.py | 6 +- moban/main.py | 4 ++ moban/mobanfile/templates.py | 51 ++++++++-------- moban/plugins/template.py | 10 ++- moban/utils.py | 52 +++++++++++----- rnd_requirements.txt | 1 + .../test_command_line_options.py | 6 +- tests/test_docs.py | 17 ++++++ tests/test_engine.py | 2 +- tests/test_utils.py | 4 +- 23 files changed, 260 insertions(+), 51 deletions(-) create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/.moban.yml create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/README.rst create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/a.output create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/cool-templates/base.jj2 create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/cool-templates/cool.template.jj2 create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/custom-config/data.base.yaml create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/custom-templates/subfolder/template.in.zip.jj2 create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/custom-templates/template.in.zip.jj2 create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/data.yml create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/data2.yml create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/templates.zip diff --git a/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml b/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml new file mode 100644 index 00000000..2593a187 --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml @@ -0,0 +1,13 @@ +configuration: + configuration_dir: 'custom-config' + template_dir: + - zip://templates.zip + - cool-templates + - '.' +targets: + - output: a.output + configuration: data.yml + template: template.in.zip.jj2 + - output: a.output2 + configuration: data2.yml + template: subfolder/template.in.zip.jj2 diff --git a/docs/level-20-templates-configs-in-zip-or-tar/README.rst b/docs/level-20-templates-configs-in-zip-or-tar/README.rst new file mode 100644 index 00000000..6858a6ef --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/README.rst @@ -0,0 +1,57 @@ +Level 20: templates, files in a zip or tar +================================================================================ + +On top of level 6, you could have files in a zip or tar. +In the following example:: + + configuration: + configuration_dir: 'custom-config' + template_dir: + - zip://templates.zip + - cool-templates + - '.' + targets: + - output: a.output + configuration: data.yml + template: template.in.zip.jj2 + - output: a.output2 + configuration: data2.yml + template: subfolder/template.in.zip.jj2 + +where `template.in.zip.jj2` were loaded from a zip file + + +Evaluation +-------------------------------------------------------------------------------- + +Please go to `docs/level-20-templates-configs-in-zip-or-tar` directory. + +Here is the command to launch it: + +.. code-block:: bash + + moban + +'a.output' is the generated file:: + + ========header============ + + world + + shijie + + this demonstrations jinja2's include statement + + ========footer============ + +`a.output2` is:: + + ========header============ + + world2 + + shijie + + this demonstrations jinja2's include statement + + ========footer============ diff --git a/docs/level-20-templates-configs-in-zip-or-tar/a.output b/docs/level-20-templates-configs-in-zip-or-tar/a.output new file mode 100644 index 00000000..9649d143 --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/a.output @@ -0,0 +1,9 @@ +========header============ + +world + +shijie + +this demonstrates jinja2's include statement + +========footer============ diff --git a/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/base.jj2 b/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/base.jj2 new file mode 100644 index 00000000..17b0361b --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/base.jj2 @@ -0,0 +1,11 @@ +{%block header %} +{%endblock%} + +{{hello}} + +{{nihao}} + +{%include 'cool.template.jj2'%} + +{%block footer %} +{%endblock%} diff --git a/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/cool.template.jj2 b/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/cool.template.jj2 new file mode 100644 index 00000000..863bf43e --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/cool.template.jj2 @@ -0,0 +1 @@ +this demonstrates jinja2's include statement diff --git a/docs/level-20-templates-configs-in-zip-or-tar/custom-config/data.base.yaml b/docs/level-20-templates-configs-in-zip-or-tar/custom-config/data.base.yaml new file mode 100644 index 00000000..3a35fadb --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/custom-config/data.base.yaml @@ -0,0 +1,2 @@ +nihao: shijie +hello: shijie diff --git a/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/subfolder/template.in.zip.jj2 b/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/subfolder/template.in.zip.jj2 new file mode 100644 index 00000000..44672595 --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/subfolder/template.in.zip.jj2 @@ -0,0 +1,9 @@ +{%extends 'base.jj2' %} + +{%block header %} +========header============ +{%endblock%} + +{%block footer %} +========footer============ +{%endblock%} diff --git a/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/template.in.zip.jj2 b/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/template.in.zip.jj2 new file mode 100644 index 00000000..44672595 --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/template.in.zip.jj2 @@ -0,0 +1,9 @@ +{%extends 'base.jj2' %} + +{%block header %} +========header============ +{%endblock%} + +{%block footer %} +========footer============ +{%endblock%} diff --git a/docs/level-20-templates-configs-in-zip-or-tar/data.yml b/docs/level-20-templates-configs-in-zip-or-tar/data.yml new file mode 100644 index 00000000..168bfb12 --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/data.yml @@ -0,0 +1,2 @@ +overrides: data.base.yaml +hello: world diff --git a/docs/level-20-templates-configs-in-zip-or-tar/data2.yml b/docs/level-20-templates-configs-in-zip-or-tar/data2.yml new file mode 100644 index 00000000..e75891e6 --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/data2.yml @@ -0,0 +1,2 @@ +overrides: data.base.yaml +hello: world2 diff --git a/docs/level-20-templates-configs-in-zip-or-tar/templates.zip b/docs/level-20-templates-configs-in-zip-or-tar/templates.zip new file mode 100644 index 0000000000000000000000000000000000000000..09308e5f622e8e21c557e5b9c87d04bcaed4ab5d GIT binary patch literal 510 zcmWIWW@Zs#U|`^2c)#hlpMLwE5Eme?4~T^sWEe_Pa|?13OH%bR^Yp4R3-q$Gj6y>= z8JI5~u!+44#HAJ742&#a85tPBhO7?rU8~`F)>|_yz~4*9@9ODOMnT~LYo9(^73O>D zr2o0F6CS=kp1Mq&H=P(=j)-wGh%)ywtaiUPYufzyrd6U!P0`v60XQ8b3v^I%X;NB# zPD*N#{(v}*kx7mjS7=E9!;FCe7!(Xk8bK_q4rGOd9a^{rc;hx$mI2w|i9mx1gaIxq SSlK|XVgkZFKw1anPzC^9I*}Iu literal 0 HcmV?d00001 diff --git a/moban/copy/__init__.py b/moban/copy/__init__.py index 1d197d1d..4a9e2deb 100644 --- a/moban/copy/__init__.py +++ b/moban/copy/__init__.py @@ -1,4 +1,4 @@ -from moban import utils, constants +from moban import utils, constants, file_system from lml.plugin import PluginInfo @@ -26,8 +26,7 @@ def get_template(self, template_file): real_file_name = utils.find_file_in_template_dirs( template_file, self.template_dirs ) - with open(real_file_name, "rb") as file_handle: - return file_handle.read() + return file_system.read_binary(real_file_name) def get_template_from_string(self, string): return string diff --git a/moban/file_system.py b/moban/file_system.py index fc0616bd..ed610254 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -27,6 +27,7 @@ def wrapper(*args, **kwds): return wrapper +@log_fs_failure @contextmanager def open_fs(path): path = to_unicode(path) @@ -41,6 +42,7 @@ def open_fs(path): the_fs.close() +@log_fs_failure def read_unicode(path): path = to_unicode(path) dir_name = fs.path.dirname(path) @@ -50,6 +52,16 @@ def read_unicode(path): return file_handle.read() +@log_fs_failure +def read_binary(path): + path = to_unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as fs_system: + return fs_system.readbytes(the_file_name) + + +@log_fs_failure def write_bytes(filename, bytes_content): filename = to_unicode(filename) dir_name = fs.path.dirname(filename) @@ -58,7 +70,13 @@ def write_bytes(filename, bytes_content): the_fs.writebytes(the_file_name, bytes_content) +@log_fs_failure def is_dir(path): + if 'zip://' in path: + zip_file, folder = path.split('.zip/') + with fs.open_fs(zip_file+'.zip') as the_fs: + return the_fs.isdir(folder) + path = to_unicode(path) dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) @@ -66,6 +84,7 @@ def is_dir(path): return the_fs.isdir(the_file_name) +@log_fs_failure def is_file(path): path = to_unicode(path) dir_name = fs.path.dirname(path) @@ -74,10 +93,18 @@ def is_file(path): return the_fs.isfile(the_file_name) +@log_fs_failure def exists(path): path = to_unicode(path) + if 'zip://' in path: + try: + with fs.open_fs(path) as the_fs: + return True + except fs.errors.CreateFailed: + return False dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) + try: with fs.open_fs(dir_name) as the_fs: return the_fs.exists(the_file_name) @@ -85,6 +112,7 @@ def exists(path): return False +@log_fs_failure def list_dir(path): path = to_unicode(path) dir_name = fs.path.dirname(path) @@ -104,6 +132,16 @@ def abspath(path): return the_fs.getsyspath(the_file_name) +@log_fs_failure +def fs_url(path): + path = to_unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + + with fs.open_fs(dir_name) as the_fs: + return the_fs.geturl(the_file_name) + + def to_unicode(path): if PY2 and path.__class__.__name__ != "unicode": return u"".__class__(path) diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index e80cd15f..74f9aedf 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -1,4 +1,5 @@ import re +import logging from importlib import import_module from moban import constants, exceptions @@ -6,6 +7,7 @@ from lml.loader import scan_plugins_regex from lml.plugin import PluginInfo, PluginManager from jinja2.exceptions import TemplateNotFound +from jinja2_fsloader import FSLoader JINJA2_LIBRARIES = "^moban_jinja2_.+$" JINJA2_EXENSIONS = [ @@ -15,6 +17,7 @@ "moban.jinja2.tests.files", ] JINJA2_THIRD_PARTY_EXTENSIONS = ["jinja2.ext.do", "jinja2.ext.loopcontrols"] +LOG = logging.getLogger(__name__) class PluginMixin: @@ -62,9 +65,10 @@ def __init__(self, template_dirs, options=None): :param list temp_dirs: a list of template directories :param dict options: a dictionary containing environmental parameters """ + LOG.debug("Jinja template dirs: %s", template_dirs) load_jinja2_extensions() self.template_dirs = template_dirs - template_loader = FileSystemLoader(template_dirs) + template_loader = FSLoader(template_dirs) env_params = dict( loader=template_loader, keep_trailing_newline=True, diff --git a/moban/main.py b/moban/main.py index 619e0a61..8e922aaf 100644 --- a/moban/main.py +++ b/moban/main.py @@ -10,6 +10,7 @@ """ import sys import argparse +import logging from moban import plugins, reporter, constants, mobanfile, exceptions from moban.utils import merge @@ -17,6 +18,8 @@ from moban.hashstore import HASH_STORE from moban.data_loaders.manager import load_data +LOG = logging.getLogger() + def main(): """ @@ -38,6 +41,7 @@ def main(): exceptions.NoThirdPartyEngine, exceptions.MobanfileGrammarException, ) as e: + LOG.exception(e) reporter.report_error_message(str(e)) moban_exit(options[constants.LABEL_EXIT_CODE], constants.ERROR) else: diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 85a4e66e..f9dc772d 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -1,3 +1,4 @@ +import fs import logging from moban import reporter, file_system @@ -40,34 +41,36 @@ def handle_template(template_file, output, template_dirs): def _list_dir_files(source, actual_source_path, dest): - for file_name in file_system.list_dir(actual_source_path): - if file_system.is_file( - file_system.path_join(actual_source_path, file_name) - ): - # please note jinja2 does NOT like windows path - # hence the following statement looks like cross platform - # src_file_under_dir = os.path.join(source, file_name) - # but actually it breaks windows instead. - src_file_under_dir = "%s/%s" % (source, file_name) + with fs.open_fs(actual_source_path) as fs_handle: + for file_name in fs_handle.listdir('.'): + if fs_handle.isfile( + file_name + ): + # please note jinja2 does NOT like windows path + # hence the following statement looks like cross platform + # src_file_under_dir = os.path.join(source, file_name) + # but actually it breaks windows instead. + src_file_under_dir = "%s/%s" % (source, file_name) - dest_file_under_dir = file_system.path_join(dest, file_name) - template_type = _get_template_type(src_file_under_dir) - yield (src_file_under_dir, dest_file_under_dir, template_type) + dest_file_under_dir = file_system.path_join(dest, file_name) + template_type = _get_template_type(src_file_under_dir) + yield (src_file_under_dir, dest_file_under_dir, template_type) def _listing_directory_files_recusively(source, actual_source_path, dest): - for file_name in file_system.list_dir(actual_source_path): - src_file_under_dir = file_system.path_join(source, file_name) - dest_file_under_dir = file_system.path_join(dest, file_name) - real_src_file = file_system.path_join(actual_source_path, file_name) - if file_system.is_file(real_src_file): - template_type = _get_template_type(src_file_under_dir) - yield (src_file_under_dir, dest_file_under_dir, template_type) - elif file_system.is_dir(real_src_file): - for a_triple in _listing_directory_files_recusively( - src_file_under_dir, real_src_file, dest_file_under_dir - ): - yield a_triple + with fs.open_fs(actual_source_path) as fs_handle: + for file_name in fs_handle.listdir('.'): + src_file_under_dir = file_system.path_join(source, file_name) + dest_file_under_dir = file_system.path_join(dest, file_name) + real_src_file = "%s/%s" % (actual_source_path, file_name) + if fs_handle.isfile(file_name): + template_type = _get_template_type(src_file_under_dir) + yield (src_file_under_dir, dest_file_under_dir, template_type) + elif fs_handle.isdir(file_name): + for a_triple in _listing_directory_files_recusively( + src_file_under_dir, real_src_file, dest_file_under_dir + ): + yield a_triple def _get_template_type(template_file): diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 03ca26d6..1d05ee9c 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -1,3 +1,4 @@ +import os import sys import logging @@ -28,6 +29,8 @@ def register_options(self, template_types): self.options_registry.update(template_types) def get_engine(self, template_type, template_dirs, context_dirs): + template_dirs = list(expand_template_directories(template_dirs)) + template_dirs = utils.verify_the_existence_of_directories(template_dirs) if template_type in self.options_registry: custom_engine_spec = self.options_registry[template_type] engine_cls = self.load_me_now( @@ -57,8 +60,6 @@ def raise_exception(self, key): class MobanEngine(object): def __init__(self, template_dirs, context_dirs, engine): - template_dirs = list(expand_template_directories(template_dirs)) - utils.verify_the_existence_of_directories(template_dirs) context_dirs = expand_template_directory(context_dirs) self.context = Context(context_dirs) self.template_dirs = template_dirs @@ -183,9 +184,12 @@ def expand_template_directory(directory): translated_directory = None if ":" in directory and directory[1] != ":" and "://" not in directory: translated_directory = deprecated_moban_path_notation(directory) + elif "://" in directory: + translated_directory = directory else: # local template path - translated_directory = file_system.abspath(directory) + translated_directory = os.path.normcase(os.path.abspath(directory)) + translated_directory = file_system.fs_url(translated_directory) return translated_directory diff --git a/moban/utils.py b/moban/utils.py index 93879e92..909f2f4f 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,4 +1,5 @@ import os +import fs import sys import stat import errno @@ -6,7 +7,7 @@ from moban import constants, exceptions, file_system -log = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -34,8 +35,15 @@ def search_file(base_dir, file_name): the_file = file_name if not file_system.exists(the_file): if base_dir: - the_file = file_system.path_join(base_dir, file_name) - if not file_system.exists(the_file): + try: + with fs.open_fs(base_dir) as fs_handle: + if fs_handle.exists(the_file): + the_file = fs_handle.geturl(file_name) + else: + raise IOError( + constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) + ) + except fs.errors.CreateFailed: raise IOError( constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) ) @@ -90,21 +98,31 @@ def pip_install(packages): def get_template_path(template_dirs, template): for a_dir in template_dirs: - template_file_exists = file_system.exists( - file_system.path_join(a_dir, template) - ) and file_system.is_file(file_system.path_join(a_dir, template)) - - if template_file_exists: - return file_system.path_join(a_dir, template) + try: + with fs.open_fs(a_dir) as fs_handle: + template_file_exists = ( + fs_handle.exists(template) + and fs_handle.isfile(template)) + + if template_file_exists: + if 'zip://' in a_dir: + return '%s/%s' % (a_dir, template) + return fs_handle.geturl(template) + except fs.errors.CreateFailed: + continue raise exceptions.FileNotFound def verify_the_existence_of_directories(dirs): + LOG.debug("Verifying the existence: %s", dirs) if not isinstance(dirs, list): dirs = [dirs] + results = [] + for directory in dirs: - if os.path.exists(directory): + if file_system.exists(directory): + results.append(directory) continue should_I_ignore = ( constants.DEFAULT_CONFIGURATION_DIRNAME in directory @@ -115,15 +133,19 @@ def verify_the_existence_of_directories(dirs): pass else: raise exceptions.DirectoryNotFound( - constants.MESSAGE_DIR_NOT_EXIST % os.path.abspath(directory) + constants.MESSAGE_DIR_NOT_EXIST % directory ) + return results def find_file_in_template_dirs(src, template_dirs): - log.debug(template_dirs) + LOG.debug(template_dirs) for folder in template_dirs: - path = file_system.path_join(folder, src) - if file_system.exists(path): - return path + with fs.open_fs(folder) as fs_handle: + if fs_handle.exists(src): + if 'zip://' in folder: + return "%s/%s" % (folder, src) + else: + return fs_handle.geturl(src) else: return None diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 0565126f..8d9551c9 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,2 +1,3 @@ https://github.com/moremoban/moban-handlebars/archive/dev.zip +https://github.com/moremoban/jinja2-fsloader/archive/dev.zip diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index abb57c34..7f652dba 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -25,9 +25,9 @@ def test_custom_options(self, fake_template_doer, fake_abspath): "-c", self.config_file, "-cd", - "/home/developer/configuration", + ".", "-td", - "/home/developer/templates", + ".", "-t", "a.jj2", ] @@ -388,7 +388,7 @@ def setUp(self): f.write("hello: world") @patch( - "moban.utils.verify_the_existence_of_directories", return_value=True + "moban.utils.verify_the_existence_of_directories", return_value='.' ) def test_single_command(self, _): test_args = ["moban"] diff --git a/tests/test_docs.py b/tests/test_docs.py index ad7b4487..526de056 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -100,6 +100,23 @@ def test_level_6(self): folder = "level-6-complex-configuration" self._raw_moban(["moban"], folder, expected, "a.output2") + def test_level_20(self): + expected = custom_dedent( + """ + ========header============ + + world2 + + shijie + + this demonstrates jinja2's include statement + + ========footer============ + """ + ) + folder = "level-20-templates-configs-in-zip-or-tar" + self._raw_moban(["moban"], folder, expected, "a.output2") + def test_level_7(self): expected = custom_dedent( """ diff --git a/tests/test_engine.py b/tests/test_engine.py index 44e3690f..2a56152f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -66,7 +66,7 @@ def test_unknown_template_type(): @raises(exceptions.DirectoryNotFound) def test_non_existent_tmpl_directries(): - MobanEngine("abc", "tests", Engine) + ENGINES.get_engine("jj2", 'idontexist', "") @raises(exceptions.DirectoryNotFound) diff --git a/tests/test_utils.py b/tests/test_utils.py index c907f073..3a0a8719 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -113,7 +113,9 @@ def test_get_template_path(): ] template = "a.jj2" template_path = get_template_path(temp_dirs, template) - expected = fs.path.join("tests", "fixtures", "template-tests", "a.jj2") + expected = 'file://' + os.path.normcase( + os.path.abspath( + os.path.join("tests", "fixtures", "template-tests", "a.jj2"))) eq_(template_path, expected) From 6a7b506a9d46f1e117069ba6870b6be9792e83b2 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 00:44:07 +0100 Subject: [PATCH 063/167] :shirt: update coding style. will do code refactoring later --- moban/file_system.py | 8 ++++---- moban/jinja2/engine.py | 2 +- moban/main.py | 2 +- moban/mobanfile/templates.py | 11 +++++------ moban/plugins/template.py | 4 +++- moban/utils.py | 18 ++++++++++-------- .../test_command_line_options.py | 4 +--- tests/test_engine.py | 2 +- tests/test_utils.py | 6 ++++-- 9 files changed, 30 insertions(+), 27 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index ed610254..b4cd5981 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -72,9 +72,9 @@ def write_bytes(filename, bytes_content): @log_fs_failure def is_dir(path): - if 'zip://' in path: - zip_file, folder = path.split('.zip/') - with fs.open_fs(zip_file+'.zip') as the_fs: + if "zip://" in path: + zip_file, folder = path.split(".zip/") + with fs.open_fs(zip_file + ".zip") as the_fs: return the_fs.isdir(folder) path = to_unicode(path) @@ -96,7 +96,7 @@ def is_file(path): @log_fs_failure def exists(path): path = to_unicode(path) - if 'zip://' in path: + if "zip://" in path: try: with fs.open_fs(path) as the_fs: return True diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index 74f9aedf..c088d17d 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -6,8 +6,8 @@ from jinja2 import Template, Environment, FileSystemLoader from lml.loader import scan_plugins_regex from lml.plugin import PluginInfo, PluginManager -from jinja2.exceptions import TemplateNotFound from jinja2_fsloader import FSLoader +from jinja2.exceptions import TemplateNotFound JINJA2_LIBRARIES = "^moban_jinja2_.+$" JINJA2_EXENSIONS = [ diff --git a/moban/main.py b/moban/main.py index 8e922aaf..75eec3d0 100644 --- a/moban/main.py +++ b/moban/main.py @@ -9,8 +9,8 @@ """ import sys -import argparse import logging +import argparse from moban import plugins, reporter, constants, mobanfile, exceptions from moban.utils import merge diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index f9dc772d..d1097931 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -1,9 +1,10 @@ -import fs import logging from moban import reporter, file_system from moban.utils import find_file_in_template_dirs +import fs + log = logging.getLogger(__name__) @@ -42,10 +43,8 @@ def handle_template(template_file, output, template_dirs): def _list_dir_files(source, actual_source_path, dest): with fs.open_fs(actual_source_path) as fs_handle: - for file_name in fs_handle.listdir('.'): - if fs_handle.isfile( - file_name - ): + for file_name in fs_handle.listdir("."): + if fs_handle.isfile(file_name): # please note jinja2 does NOT like windows path # hence the following statement looks like cross platform # src_file_under_dir = os.path.join(source, file_name) @@ -59,7 +58,7 @@ def _list_dir_files(source, actual_source_path, dest): def _listing_directory_files_recusively(source, actual_source_path, dest): with fs.open_fs(actual_source_path) as fs_handle: - for file_name in fs_handle.listdir('.'): + for file_name in fs_handle.listdir("."): src_file_under_dir = file_system.path_join(source, file_name) dest_file_under_dir = file_system.path_join(dest, file_name) real_src_file = "%s/%s" % (actual_source_path, file_name) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 1d05ee9c..467e34b6 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -30,7 +30,9 @@ def register_options(self, template_types): def get_engine(self, template_type, template_dirs, context_dirs): template_dirs = list(expand_template_directories(template_dirs)) - template_dirs = utils.verify_the_existence_of_directories(template_dirs) + template_dirs = utils.verify_the_existence_of_directories( + template_dirs + ) if template_type in self.options_registry: custom_engine_spec = self.options_registry[template_type] engine_cls = self.load_me_now( diff --git a/moban/utils.py b/moban/utils.py index 909f2f4f..c186fa9a 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,5 +1,4 @@ import os -import fs import sys import stat import errno @@ -7,6 +6,8 @@ from moban import constants, exceptions, file_system +import fs + LOG = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -41,7 +42,8 @@ def search_file(base_dir, file_name): the_file = fs_handle.geturl(file_name) else: raise IOError( - constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) + constants.ERROR_DATA_FILE_NOT_FOUND + % (file_name, the_file) ) except fs.errors.CreateFailed: raise IOError( @@ -100,13 +102,13 @@ def get_template_path(template_dirs, template): for a_dir in template_dirs: try: with fs.open_fs(a_dir) as fs_handle: - template_file_exists = ( - fs_handle.exists(template) - and fs_handle.isfile(template)) + template_file_exists = fs_handle.exists( + template + ) and fs_handle.isfile(template) if template_file_exists: - if 'zip://' in a_dir: - return '%s/%s' % (a_dir, template) + if "zip://" in a_dir: + return "%s/%s" % (a_dir, template) return fs_handle.geturl(template) except fs.errors.CreateFailed: continue @@ -143,7 +145,7 @@ def find_file_in_template_dirs(src, template_dirs): for folder in template_dirs: with fs.open_fs(folder) as fs_handle: if fs_handle.exists(src): - if 'zip://' in folder: + if "zip://" in folder: return "%s/%s" % (folder, src) else: return fs_handle.geturl(src) diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index 7f652dba..c6010599 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -387,9 +387,7 @@ def setUp(self): with open(self.data_file, "w") as f: f.write("hello: world") - @patch( - "moban.utils.verify_the_existence_of_directories", return_value='.' - ) + @patch("moban.utils.verify_the_existence_of_directories", return_value=".") def test_single_command(self, _): test_args = ["moban"] with patch.object(sys, "argv", test_args): diff --git a/tests/test_engine.py b/tests/test_engine.py index 2a56152f..114bac97 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -66,7 +66,7 @@ def test_unknown_template_type(): @raises(exceptions.DirectoryNotFound) def test_non_existent_tmpl_directries(): - ENGINES.get_engine("jj2", 'idontexist', "") + ENGINES.get_engine("jj2", "idontexist", "") @raises(exceptions.DirectoryNotFound) diff --git a/tests/test_utils.py b/tests/test_utils.py index 3a0a8719..1b95d66a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -113,9 +113,11 @@ def test_get_template_path(): ] template = "a.jj2" template_path = get_template_path(temp_dirs, template) - expected = 'file://' + os.path.normcase( + expected = "file://" + os.path.normcase( os.path.abspath( - os.path.join("tests", "fixtures", "template-tests", "a.jj2"))) + os.path.join("tests", "fixtures", "template-tests", "a.jj2") + ) + ) eq_(template_path, expected) From 04dd0dcff1a08ef3d1d7aa05a33a922b074ba94b Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 00:49:21 +0100 Subject: [PATCH 064/167] :fire: use registry.install instead of setup.py entry points --- .azure-pipelines-steps-macos.yml | 1 - .azure-pipelines-steps.yml | 1 - .moban.cd/moban.yml | 3 --- moban/fs_openers.py | 3 +++ moban/main.py | 1 + setup.py | 4 ---- test.sh | 2 +- 7 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.azure-pipelines-steps-macos.yml b/.azure-pipelines-steps-macos.yml index 0198ca11..8958395b 100644 --- a/.azure-pipelines-steps-macos.yml +++ b/.azure-pipelines-steps-macos.yml @@ -6,7 +6,6 @@ steps: test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt pip install -r requirements.txt pip install -r tests/requirements.txt - python setup.py develop displayName: 'Setup dependencies' - script: | make diff --git a/.azure-pipelines-steps.yml b/.azure-pipelines-steps.yml index 68783aa0..2ef569c6 100644 --- a/.azure-pipelines-steps.yml +++ b/.azure-pipelines-steps.yml @@ -6,7 +6,6 @@ steps: if exist rnd_requirements.txt pip install -r rnd_requirements.txt pip install -r requirements.txt pip install -r tests\requirements.txt - python setup.py develop displayName: 'Setup dependencies' - script: | test.bat diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 63ada6fe..9db6158e 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -12,9 +12,6 @@ master: index entry_points: console_scripts: - "moban = moban.main:main" - fs.opener: - - "pypi = moban.fs_openers:PypiFSOpener" - - "repo = moban.fs_openers:RepoFSOpener" company: Onni Software Ltd. copyright_year: 2017-2019 copyright: 2017-2019 Onni Software Ltd. and its contributors diff --git a/moban/fs_openers.py b/moban/fs_openers.py index 7ab47651..39145283 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -4,8 +4,10 @@ import fs.path from fs.osfs import OSFS from fs.opener import Opener +from fs.opener.registry import registry +@registry.install class PypiFSOpener(Opener): protocols = ["pypi"] @@ -17,6 +19,7 @@ def open_fs(self, fs_url, parse_result, writeable, create, cwd): return osfs +@registry.install class RepoFSOpener(Opener): protocols = ["repo"] diff --git a/moban/main.py b/moban/main.py index 75eec3d0..1bb888f7 100644 --- a/moban/main.py +++ b/moban/main.py @@ -17,6 +17,7 @@ from moban._version import __version__ from moban.hashstore import HASH_STORE from moban.data_loaders.manager import load_data +import moban.fs_openers # noqa: F401 LOG = logging.getLogger() diff --git a/setup.py b/setup.py index 49706583..63982d39 100644 --- a/setup.py +++ b/setup.py @@ -45,10 +45,6 @@ "console_scripts": [ "moban = moban.main:main", ], - "fs.opener": [ - "pypi = moban.fs_openers:PypiFSOpener", - "repo = moban.fs_openers:RepoFSOpener", - ], } DESCRIPTION = ( "Yet another jinja2 cli command for static text generation" diff --git a/test.sh b/test.sh index a2a61ade..b294f53a 100644 --- a/test.sh +++ b/test.sh @@ -1,3 +1,3 @@ pip freeze -python setup.py develop + nosetests --with-cov --with-doctest --doctest-extension=.rst --cover-package moban --cover-package tests From 6255fe21c19dcdf9407db793b68a1eeab75d4232 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 00:51:56 +0100 Subject: [PATCH 065/167] :bug: install master branch of custom jinja2-fsloader --- rnd_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 8d9551c9..45f5f008 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,3 +1,3 @@ https://github.com/moremoban/moban-handlebars/archive/dev.zip -https://github.com/moremoban/jinja2-fsloader/archive/dev.zip +https://github.com/moremoban/jinja2-fsloader/archive/master.zip From 818ea5b7e90e9fafd508ea54a0f9197a1923fef7 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 08:48:07 +0100 Subject: [PATCH 066/167] :hammer: code refactoring --- moban/data_loaders/json_loader.py | 4 ++-- moban/data_loaders/yaml.py | 4 ++-- moban/file_system.py | 13 ++++++++++++- moban/mobanfile/templates.py | 6 ++---- moban/utils.py | 13 ++++--------- rnd_requirements.txt | 3 +++ 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/moban/data_loaders/json_loader.py b/moban/data_loaders/json_loader.py index 15e2d4ec..6393af47 100644 --- a/moban/data_loaders/json_loader.py +++ b/moban/data_loaders/json_loader.py @@ -2,7 +2,7 @@ from moban import constants from lml.plugin import PluginInfo -from moban.file_system import open_fs +from moban.file_system import open_file @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["json"]) @@ -10,6 +10,6 @@ def open_json(file_name): """ returns json contents as string """ - with open_fs(file_name) as json_file: + with open_file(file_name) as json_file: data = json.load(json_file) return data diff --git a/moban/data_loaders/yaml.py b/moban/data_loaders/yaml.py index 9d051f85..f082730c 100644 --- a/moban/data_loaders/yaml.py +++ b/moban/data_loaders/yaml.py @@ -1,12 +1,12 @@ from moban import constants from lml.plugin import PluginInfo from ruamel.yaml import YAML -from moban.file_system import open_fs +from moban.file_system import open_file @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["yaml", "yml"]) def open_yaml(file_name): - with open_fs(file_name) as data_yaml: + with open_file(file_name) as data_yaml: yaml = YAML(typ="rt") data = yaml.load(data_yaml) return data diff --git a/moban/file_system.py b/moban/file_system.py index b4cd5981..7787956f 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -29,7 +29,7 @@ def wrapper(*args, **kwds): @log_fs_failure @contextmanager -def open_fs(path): +def open_file(path): path = to_unicode(path) dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) @@ -42,6 +42,17 @@ def open_fs(path): the_fs.close() +@log_fs_failure +@contextmanager +def open_fs(path): + path = to_unicode(path) + the_fs = fs.open_fs(path) + try: + yield the_fs + finally: + the_fs.close() + + @log_fs_failure def read_unicode(path): path = to_unicode(path) diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index d1097931..ae9c4174 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -3,8 +3,6 @@ from moban import reporter, file_system from moban.utils import find_file_in_template_dirs -import fs - log = logging.getLogger(__name__) @@ -42,7 +40,7 @@ def handle_template(template_file, output, template_dirs): def _list_dir_files(source, actual_source_path, dest): - with fs.open_fs(actual_source_path) as fs_handle: + with file_system.open_fs(actual_source_path) as fs_handle: for file_name in fs_handle.listdir("."): if fs_handle.isfile(file_name): # please note jinja2 does NOT like windows path @@ -57,7 +55,7 @@ def _list_dir_files(source, actual_source_path, dest): def _listing_directory_files_recusively(source, actual_source_path, dest): - with fs.open_fs(actual_source_path) as fs_handle: + with file_system.open_fs(actual_source_path) as fs_handle: for file_name in fs_handle.listdir("."): src_file_under_dir = file_system.path_join(source, file_name) dest_file_under_dir = file_system.path_join(dest, file_name) diff --git a/moban/utils.py b/moban/utils.py index c186fa9a..e4b75810 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -37,7 +37,7 @@ def search_file(base_dir, file_name): if not file_system.exists(the_file): if base_dir: try: - with fs.open_fs(base_dir) as fs_handle: + with file_system.open_fs(base_dir) as fs_handle: if fs_handle.exists(the_file): the_file = fs_handle.geturl(file_name) else: @@ -101,14 +101,12 @@ def pip_install(packages): def get_template_path(template_dirs, template): for a_dir in template_dirs: try: - with fs.open_fs(a_dir) as fs_handle: + with file_system.open_fs(a_dir) as fs_handle: template_file_exists = fs_handle.exists( template ) and fs_handle.isfile(template) if template_file_exists: - if "zip://" in a_dir: - return "%s/%s" % (a_dir, template) return fs_handle.geturl(template) except fs.errors.CreateFailed: continue @@ -143,11 +141,8 @@ def verify_the_existence_of_directories(dirs): def find_file_in_template_dirs(src, template_dirs): LOG.debug(template_dirs) for folder in template_dirs: - with fs.open_fs(folder) as fs_handle: + with file_system.open_fs(folder) as fs_handle: if fs_handle.exists(src): - if "zip://" in folder: - return "%s/%s" % (folder, src) - else: - return fs_handle.geturl(src) + return fs_handle.geturl(src) else: return None diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 45f5f008..90e15c68 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,3 +1,6 @@ https://github.com/moremoban/moban-handlebars/archive/dev.zip https://github.com/moremoban/jinja2-fsloader/archive/master.zip +https://github.com/moremoban/pyfilesystem2/archive/master.zip + + From 7f20b1bcf4a8aa0f5f03256c34deac5b21df65fb Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 08:51:45 +0100 Subject: [PATCH 067/167] :bug: tuning the sequence of rnd installation --- rnd_requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 90e15c68..5b52dddd 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,6 +1,6 @@ -https://github.com/moremoban/moban-handlebars/archive/dev.zip -https://github.com/moremoban/jinja2-fsloader/archive/master.zip https://github.com/moremoban/pyfilesystem2/archive/master.zip +https://github.com/moremoban/jinja2-fsloader/archive/master.zip +https://github.com/moremoban/moban-handlebars/archive/dev.zip From caf3d985a955170d1ded54682a4c2127b217baca Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 09:01:55 +0100 Subject: [PATCH 068/167] :shirt: code reformatting and :bug: fix python 2 unicode problem --- moban/fs_openers.py | 5 ++--- moban/main.py | 2 +- moban/mobanfile/templates.py | 10 +++++----- moban/utils.py | 7 +++---- tests/mobanfile/test_mobanfile.py | 3 +-- tests/mobanfile/test_targets.py | 3 +-- tests/mobanfile/test_templates.py | 3 +-- tests/test_context.py | 3 +-- tests/test_copy_encoding.py | 3 +-- tests/test_engine.py | 3 +-- tests/test_json_loader.py | 3 +-- tests/test_regression.py | 3 +-- tests/test_repo.py | 3 +-- tests/test_template.py | 3 +-- tests/test_utils.py | 3 +-- tests/test_yaml_loader.py | 3 +-- 16 files changed, 23 insertions(+), 37 deletions(-) diff --git a/moban/fs_openers.py b/moban/fs_openers.py index 39145283..3897cfa7 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -1,10 +1,9 @@ -from moban import repo -from moban.plugins.library import LIBRARIES - import fs.path +from moban import repo from fs.osfs import OSFS from fs.opener import Opener from fs.opener.registry import registry +from moban.plugins.library import LIBRARIES @registry.install diff --git a/moban/main.py b/moban/main.py index 1bb888f7..7ad010cc 100644 --- a/moban/main.py +++ b/moban/main.py @@ -12,12 +12,12 @@ import logging import argparse +import moban.fs_openers # noqa: F401 from moban import plugins, reporter, constants, mobanfile, exceptions from moban.utils import merge from moban._version import __version__ from moban.hashstore import HASH_STORE from moban.data_loaders.manager import load_data -import moban.fs_openers # noqa: F401 LOG = logging.getLogger() diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index ae9c4174..8c8fccd9 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -41,8 +41,8 @@ def handle_template(template_file, output, template_dirs): def _list_dir_files(source, actual_source_path, dest): with file_system.open_fs(actual_source_path) as fs_handle: - for file_name in fs_handle.listdir("."): - if fs_handle.isfile(file_name): + for file_name in fs_handle.listdir(u"."): + if fs_handle.isfile(file_system.to_unicode(file_name)): # please note jinja2 does NOT like windows path # hence the following statement looks like cross platform # src_file_under_dir = os.path.join(source, file_name) @@ -56,14 +56,14 @@ def _list_dir_files(source, actual_source_path, dest): def _listing_directory_files_recusively(source, actual_source_path, dest): with file_system.open_fs(actual_source_path) as fs_handle: - for file_name in fs_handle.listdir("."): + for file_name in fs_handle.listdir(u"."): src_file_under_dir = file_system.path_join(source, file_name) dest_file_under_dir = file_system.path_join(dest, file_name) real_src_file = "%s/%s" % (actual_source_path, file_name) - if fs_handle.isfile(file_name): + if fs_handle.isfile(file_system.to_unicode(file_name)): template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) - elif fs_handle.isdir(file_name): + elif fs_handle.isdir(file_system.to_unicode(file_name)): for a_triple in _listing_directory_files_recusively( src_file_under_dir, real_src_file, dest_file_under_dir ): diff --git a/moban/utils.py b/moban/utils.py index e4b75810..1b785bf4 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -4,9 +4,8 @@ import errno import logging -from moban import constants, exceptions, file_system - import fs +from moban import constants, exceptions, file_system LOG = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -38,7 +37,7 @@ def search_file(base_dir, file_name): if base_dir: try: with file_system.open_fs(base_dir) as fs_handle: - if fs_handle.exists(the_file): + if fs_handle.exists(file_system.to_unicode(the_file)): the_file = fs_handle.geturl(file_name) else: raise IOError( @@ -142,7 +141,7 @@ def find_file_in_template_dirs(src, template_dirs): LOG.debug(template_dirs) for folder in template_dirs: with file_system.open_fs(folder) as fs_handle: - if fs_handle.exists(src): + if fs_handle.exists(file_system.to_unicode(src)): return fs_handle.geturl(src) else: return None diff --git a/tests/mobanfile/test_mobanfile.py b/tests/mobanfile/test_mobanfile.py index 321b8e2f..1eb3143e 100644 --- a/tests/mobanfile/test_mobanfile.py +++ b/tests/mobanfile/test_mobanfile.py @@ -1,9 +1,8 @@ +import fs.path from mock import patch from nose.tools import eq_ from moban.definitions import GitRequire, TemplateTarget -import fs.path - class TestFinder: def setUp(self): diff --git a/tests/mobanfile/test_targets.py b/tests/mobanfile/test_targets.py index 18b7b93a..dff5c850 100644 --- a/tests/mobanfile/test_targets.py +++ b/tests/mobanfile/test_targets.py @@ -1,12 +1,11 @@ import uuid +import fs.path from nose.tools import eq_, raises from moban.mobanfile import targets from moban.exceptions import GroupTargetNotFound from moban.definitions import TemplateTarget -import fs.path - TEMPLATE = "a.jj2" OUTPUT = "a.output" CONFIGURATION = "data.config" diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index 61b69479..7b362d10 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -1,9 +1,8 @@ +import fs.path from mock import patch from nose.tools import eq_ from moban.mobanfile.templates import handle_template -import fs.path - class TestHandleTemplateFunction: def setUp(self): diff --git a/tests/test_context.py b/tests/test_context.py index e74019c9..72575582 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,10 +1,9 @@ import os +import fs.path from nose.tools import eq_ from moban.plugins.context import Context -import fs.path - def test_context(): context = Context(fs.path.join("tests", "fixtures")) diff --git a/tests/test_copy_encoding.py b/tests/test_copy_encoding.py index 50880ef8..bcad3674 100644 --- a/tests/test_copy_encoding.py +++ b/tests/test_copy_encoding.py @@ -1,8 +1,7 @@ +import fs.path from moban.copy import ContentForwardEngine from nose.tools import eq_ -import fs.path - class TestCopyEncoding: def setUp(self): diff --git a/tests/test_engine.py b/tests/test_engine.py index 114bac97..0e417e14 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,6 +1,7 @@ import os import sys +import fs.path import moban.exceptions as exceptions from mock import patch from lml.plugin import PluginInfo @@ -14,8 +15,6 @@ from moban.plugins.context import Context from moban.plugins.template import MobanEngine, expand_template_directories -import fs.path - USER_HOME = fs.path.join("user", "home", ".moban", "repos") diff --git a/tests/test_json_loader.py b/tests/test_json_loader.py index 39c00e77..44ed74fa 100644 --- a/tests/test_json_loader.py +++ b/tests/test_json_loader.py @@ -1,8 +1,7 @@ +import fs.path from nose.tools import eq_ from moban.data_loaders.json_loader import open_json -import fs.path - def test_open_json(): content = open_json(fs.path.join("tests", "fixtures", "child.json")) diff --git a/tests/test_regression.py b/tests/test_regression.py index 894267e7..37b771d4 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -3,12 +3,11 @@ import filecmp from textwrap import dedent +import fs.path from mock import patch from moban.main import main from nose.tools import eq_ -import fs.path - def custom_dedent(long_texts): refined = dedent(long_texts) diff --git a/tests/test_repo.py b/tests/test_repo.py index 803e7a82..0041cc4e 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,3 +1,4 @@ +import fs.path from mock import patch from moban.repo import ( git_clone, @@ -9,8 +10,6 @@ from moban.exceptions import NoGitCommand from moban.definitions import GitRequire -import fs.path - @patch("appdirs.user_cache_dir", return_value="root") @patch("moban.utils.mkdir_p") diff --git a/tests/test_template.py b/tests/test_template.py index f4184432..091f9f0c 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -1,5 +1,6 @@ import os +import fs.path from mock import patch from nose.tools import eq_ from moban.plugins import ENGINES @@ -7,8 +8,6 @@ from moban.jinja2.engine import Engine from moban.data_loaders.yaml import open_yaml -import fs.path - MODULE = "moban.plugins.template" diff --git a/tests/test_utils.py b/tests/test_utils.py index 1b95d66a..79c0145d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,6 +3,7 @@ import stat from shutil import rmtree +import fs.path from mock import patch from nose import SkipTest from nose.tools import eq_, raises @@ -15,8 +16,6 @@ ) from moban.exceptions import FileNotFound -import fs.path - def create_file(test_file, permission): with open(test_file, "w") as f: diff --git a/tests/test_yaml_loader.py b/tests/test_yaml_loader.py index ce453c8b..c8f2c8b1 100644 --- a/tests/test_yaml_loader.py +++ b/tests/test_yaml_loader.py @@ -1,9 +1,8 @@ +import fs.path from nose.tools import eq_, raises from moban.data_loaders.yaml import open_yaml from moban.data_loaders.manager import load_data -import fs.path - def test_simple_yaml(): test_file = fs.path.join("tests", "fixtures", "simple.yaml") From a0bd87fa721193c2f8c29da82a6392fa8a10b420 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 09:40:03 +0100 Subject: [PATCH 069/167] :hammer: convert all file names at entry to unicode --- moban/definitions.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/moban/definitions.py b/moban/definitions.py index 8ae4ea45..74bf91a4 100644 --- a/moban/definitions.py +++ b/moban/definitions.py @@ -1,6 +1,8 @@ import os +import sys from moban import constants +PY2 = sys.version_info[0] == 2 class GitRequire(object): @@ -43,9 +45,9 @@ def __init__( output, template_type=constants.DEFAULT_TEMPLATE_TYPE, ): - self.template_file = template_file - self.data_file = data_file - self.original_output = output + self.template_file = to_unicode(template_file) + self.data_file = to_unicode(data_file) + self.original_output = to_unicode(output) self.template_type = template_type self.output = self.original_output @@ -53,7 +55,7 @@ def __init__( def set_template_type(self, new_template_type): self.template_type = new_template_type - if self.original_output.endswith(self.template_type): + if self.original_output.endswith(to_unicode(self.template_type)): self.output, _ = os.path.splitext(self.original_output) else: self.output = self.original_output @@ -73,3 +75,9 @@ def __repr__(self): self.output, self.template_type, ) + + +def to_unicode(path): + if PY2 and path.__class__.__name__ != "unicode": + return u"".__class__(path) + return path From 71c8fec6fb7d988a16d551a79b545dfb77ba283e Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 10:48:10 +0100 Subject: [PATCH 070/167] :bug: fix get_template non unicode problem :wheel_chair: unicode path in pyfilesystem2 is a pain for python 2 --- moban/plugins/template.py | 2 +- moban/utils.py | 1 + tests/test_utils.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 467e34b6..dfe14bba 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -83,7 +83,7 @@ def number_of_templated_files(self): def render_to_file(self, template_file, data_file, output_file): self.file_count = 1 data = self.context.get_data(data_file) - template = self.engine.get_template(template_file) + template = self.engine.get_template(file_system.to_unicode(template_file)) template_abs_path = utils.get_template_path( self.template_dirs, template_file ) diff --git a/moban/utils.py b/moban/utils.py index 1b785bf4..392761e0 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -101,6 +101,7 @@ def get_template_path(template_dirs, template): for a_dir in template_dirs: try: with file_system.open_fs(a_dir) as fs_handle: + template = file_system.to_unicode(template) template_file_exists = fs_handle.exists( template ) and fs_handle.isfile(template) diff --git a/tests/test_utils.py b/tests/test_utils.py index 79c0145d..a8f67f8a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -117,7 +117,7 @@ def test_get_template_path(): os.path.join("tests", "fixtures", "template-tests", "a.jj2") ) ) - eq_(template_path, expected) + eq_(template_path.lower(), expected.replace('\\', '/').lower()) @patch("subprocess.check_call") From 45a6bd43f5c02168f711b3d4d57bbe3987eff1b5 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 18:17:02 +0100 Subject: [PATCH 071/167] :shirt: update coding style --- moban/definitions.py | 1 + moban/jinja2/engine.py | 2 +- moban/plugins/template.py | 4 +++- tests/test_utils.py | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/moban/definitions.py b/moban/definitions.py index 74bf91a4..b2666add 100644 --- a/moban/definitions.py +++ b/moban/definitions.py @@ -2,6 +2,7 @@ import sys from moban import constants + PY2 = sys.version_info[0] == 2 diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index c088d17d..87c01b57 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -3,7 +3,7 @@ from importlib import import_module from moban import constants, exceptions -from jinja2 import Template, Environment, FileSystemLoader +from jinja2 import Template, Environment from lml.loader import scan_plugins_regex from lml.plugin import PluginInfo, PluginManager from jinja2_fsloader import FSLoader diff --git a/moban/plugins/template.py b/moban/plugins/template.py index dfe14bba..882e2927 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -83,7 +83,9 @@ def number_of_templated_files(self): def render_to_file(self, template_file, data_file, output_file): self.file_count = 1 data = self.context.get_data(data_file) - template = self.engine.get_template(file_system.to_unicode(template_file)) + template = self.engine.get_template( + file_system.to_unicode(template_file) + ) template_abs_path = utils.get_template_path( self.template_dirs, template_file ) diff --git a/tests/test_utils.py b/tests/test_utils.py index a8f67f8a..535db5d2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -117,7 +117,7 @@ def test_get_template_path(): os.path.join("tests", "fixtures", "template-tests", "a.jj2") ) ) - eq_(template_path.lower(), expected.replace('\\', '/').lower()) + eq_(template_path.lower(), expected.replace("\\", "/").lower()) @patch("subprocess.check_call") From 150e6aa53e05d44fb7c4955ab14e350936f65e22 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 20:00:48 +0100 Subject: [PATCH 072/167] :hammer: backward compactibility --- moban/adapter/__init__.py | 0 moban/adapter/osfs.py | 11 +++++++++++ moban/adapter/zipfs.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 moban/adapter/__init__.py create mode 100644 moban/adapter/osfs.py create mode 100644 moban/adapter/zipfs.py diff --git a/moban/adapter/__init__.py b/moban/adapter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/moban/adapter/osfs.py b/moban/adapter/osfs.py new file mode 100644 index 00000000..df2ffa3e --- /dev/null +++ b/moban/adapter/osfs.py @@ -0,0 +1,11 @@ +from fs.osfs import OSFS + + +class EnhancedOSFS(OSFS): + + def geturl(self, path, purpose="download"): + # type: (Text, Text) -> Text + if purpose != "download": + raise NoURL(path, purpose) + # file://D:\a\1\s\tests\fixtures\template can be not used by osfs itself. + return "file://" + self.getsyspath(path).replace('\\', '/') diff --git a/moban/adapter/zipfs.py b/moban/adapter/zipfs.py new file mode 100644 index 00000000..3a4a202f --- /dev/null +++ b/moban/adapter/zipfs.py @@ -0,0 +1,30 @@ +import zipfile + +from fs.zipfs import ZipFS +from fs.zipfs import ReadZipFS, WriteZipFS + + +class EnhancedReadZipFS(ReadZipFS): + + def geturl(self, path, purpose='download'): + return "zip://%s/%s" % (self._file, path) + + +class EnhancedZipFS(ZipFS): + def __new__( + cls, + file, # type: Union[Text, BinaryIO] + write=False, # type: bool + compression=zipfile.ZIP_DEFLATED, # type: int + encoding="utf-8", # type: Text + temp_fs="temp://__ziptemp__", # type: Text + ): + # type: (...) -> FS + # This magic returns a different class instance based on the + # value of the ``write`` parameter. + if write: + return WriteZipFS( + file, compression=compression, encoding=encoding, temp_fs=temp_fs + ) + else: + return EnhancedReadZipFS(file, encoding=encoding) From c6cf5c7cac91532916e66dafd0d99ab055fe158b Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 20:00:58 +0100 Subject: [PATCH 073/167] :shirt: coding style update --- moban/adapter/osfs.py | 3 +- moban/adapter/zipfs.py | 11 +++--- moban/fs_openers.py | 65 ++++++++++++++++++++++++++++++- moban/utils.py | 3 +- rnd_requirements.txt | 1 - tests/mobanfile/test_mobanfile.py | 3 +- tests/mobanfile/test_targets.py | 3 +- tests/mobanfile/test_templates.py | 3 +- tests/test_context.py | 3 +- tests/test_copy_encoding.py | 3 +- tests/test_engine.py | 3 +- tests/test_json_loader.py | 3 +- tests/test_regression.py | 3 +- tests/test_repo.py | 3 +- tests/test_template.py | 3 +- tests/test_utils.py | 3 +- tests/test_yaml_loader.py | 3 +- 17 files changed, 96 insertions(+), 23 deletions(-) diff --git a/moban/adapter/osfs.py b/moban/adapter/osfs.py index df2ffa3e..77801327 100644 --- a/moban/adapter/osfs.py +++ b/moban/adapter/osfs.py @@ -2,10 +2,9 @@ class EnhancedOSFS(OSFS): - def geturl(self, path, purpose="download"): # type: (Text, Text) -> Text if purpose != "download": raise NoURL(path, purpose) # file://D:\a\1\s\tests\fixtures\template can be not used by osfs itself. - return "file://" + self.getsyspath(path).replace('\\', '/') + return "file://" + self.getsyspath(path).replace("\\", "/") diff --git a/moban/adapter/zipfs.py b/moban/adapter/zipfs.py index 3a4a202f..f1591bca 100644 --- a/moban/adapter/zipfs.py +++ b/moban/adapter/zipfs.py @@ -1,12 +1,10 @@ import zipfile -from fs.zipfs import ZipFS -from fs.zipfs import ReadZipFS, WriteZipFS +from fs.zipfs import ZipFS, ReadZipFS, WriteZipFS class EnhancedReadZipFS(ReadZipFS): - - def geturl(self, path, purpose='download'): + def geturl(self, path, purpose="download"): return "zip://%s/%s" % (self._file, path) @@ -24,7 +22,10 @@ def __new__( # value of the ``write`` parameter. if write: return WriteZipFS( - file, compression=compression, encoding=encoding, temp_fs=temp_fs + file, + compression=compression, + encoding=encoding, + temp_fs=temp_fs, ) else: return EnhancedReadZipFS(file, encoding=encoding) diff --git a/moban/fs_openers.py b/moban/fs_openers.py index 3897cfa7..a78b137e 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -1,9 +1,15 @@ -import fs.path from moban import repo +from moban.adapter.osfs import EnhancedOSFS +from moban.adapter.zipfs import EnhancedZipFS +from moban.plugins.library import LIBRARIES + +import fs +import fs.path from fs.osfs import OSFS from fs.opener import Opener from fs.opener.registry import registry -from moban.plugins.library import LIBRARIES + +# FS_VERSION = _version_count(fs.__version__) @registry.install @@ -28,3 +34,58 @@ def open_fs(self, fs_url, parse_result, writeable, create, cwd): root_path = fs.path.join(actual_repo_path, dir_path) osfs = OSFS(root_path=root_path) return osfs + + +@registry.install +class ZipOpener(Opener): + """`ZipFS` opener. + """ + + protocols = ["zip"] + + def open_fs( + self, + fs_url, # type: Text + parse_result, # type: ParseResult + writeable, # type: bool + create, # type: bool + cwd, # type: Text + ): + if not create and writeable: + raise fs.errors.NotWriteable( + "Unable to open existing ZIP file for writing" + ) + zip_fs = EnhancedZipFS(parse_result.resource, write=create) + return zip_fs + + +@registry.install +class OSFSOpener(Opener): + """`OSFS` opener. + """ + + protocols = ["file", "osfs"] + + def open_fs( + self, + fs_url, # type: Text + parse_result, # type: ParseResult + writeable, # type: bool + create, # type: bool + cwd, # type: Text + ): + # type: (...) -> OSFS + from os.path import abspath, expanduser, normpath, join + + _path = abspath(join(cwd, expanduser(parse_result.resource))) + path = normpath(_path) + osfs = EnhancedOSFS(path, create=create) + return osfs + + +def _version_count(version): + tokens = version.split(".") + sum = 0 + for index, token in enumerate(reversed(tokens)): + sum = sum + int(token) * (10 ** index) + return sum diff --git a/moban/utils.py b/moban/utils.py index 392761e0..128ad98b 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -4,9 +4,10 @@ import errno import logging -import fs from moban import constants, exceptions, file_system +import fs + LOG = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 5b52dddd..52963a4d 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,4 +1,3 @@ -https://github.com/moremoban/pyfilesystem2/archive/master.zip https://github.com/moremoban/jinja2-fsloader/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip diff --git a/tests/mobanfile/test_mobanfile.py b/tests/mobanfile/test_mobanfile.py index 1eb3143e..321b8e2f 100644 --- a/tests/mobanfile/test_mobanfile.py +++ b/tests/mobanfile/test_mobanfile.py @@ -1,8 +1,9 @@ -import fs.path from mock import patch from nose.tools import eq_ from moban.definitions import GitRequire, TemplateTarget +import fs.path + class TestFinder: def setUp(self): diff --git a/tests/mobanfile/test_targets.py b/tests/mobanfile/test_targets.py index dff5c850..18b7b93a 100644 --- a/tests/mobanfile/test_targets.py +++ b/tests/mobanfile/test_targets.py @@ -1,11 +1,12 @@ import uuid -import fs.path from nose.tools import eq_, raises from moban.mobanfile import targets from moban.exceptions import GroupTargetNotFound from moban.definitions import TemplateTarget +import fs.path + TEMPLATE = "a.jj2" OUTPUT = "a.output" CONFIGURATION = "data.config" diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index 7b362d10..61b69479 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -1,8 +1,9 @@ -import fs.path from mock import patch from nose.tools import eq_ from moban.mobanfile.templates import handle_template +import fs.path + class TestHandleTemplateFunction: def setUp(self): diff --git a/tests/test_context.py b/tests/test_context.py index 72575582..e74019c9 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,9 +1,10 @@ import os -import fs.path from nose.tools import eq_ from moban.plugins.context import Context +import fs.path + def test_context(): context = Context(fs.path.join("tests", "fixtures")) diff --git a/tests/test_copy_encoding.py b/tests/test_copy_encoding.py index bcad3674..50880ef8 100644 --- a/tests/test_copy_encoding.py +++ b/tests/test_copy_encoding.py @@ -1,7 +1,8 @@ -import fs.path from moban.copy import ContentForwardEngine from nose.tools import eq_ +import fs.path + class TestCopyEncoding: def setUp(self): diff --git a/tests/test_engine.py b/tests/test_engine.py index 0e417e14..114bac97 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,7 +1,6 @@ import os import sys -import fs.path import moban.exceptions as exceptions from mock import patch from lml.plugin import PluginInfo @@ -15,6 +14,8 @@ from moban.plugins.context import Context from moban.plugins.template import MobanEngine, expand_template_directories +import fs.path + USER_HOME = fs.path.join("user", "home", ".moban", "repos") diff --git a/tests/test_json_loader.py b/tests/test_json_loader.py index 44ed74fa..39c00e77 100644 --- a/tests/test_json_loader.py +++ b/tests/test_json_loader.py @@ -1,7 +1,8 @@ -import fs.path from nose.tools import eq_ from moban.data_loaders.json_loader import open_json +import fs.path + def test_open_json(): content = open_json(fs.path.join("tests", "fixtures", "child.json")) diff --git a/tests/test_regression.py b/tests/test_regression.py index 37b771d4..894267e7 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -3,11 +3,12 @@ import filecmp from textwrap import dedent -import fs.path from mock import patch from moban.main import main from nose.tools import eq_ +import fs.path + def custom_dedent(long_texts): refined = dedent(long_texts) diff --git a/tests/test_repo.py b/tests/test_repo.py index 0041cc4e..803e7a82 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,4 +1,3 @@ -import fs.path from mock import patch from moban.repo import ( git_clone, @@ -10,6 +9,8 @@ from moban.exceptions import NoGitCommand from moban.definitions import GitRequire +import fs.path + @patch("appdirs.user_cache_dir", return_value="root") @patch("moban.utils.mkdir_p") diff --git a/tests/test_template.py b/tests/test_template.py index 091f9f0c..f4184432 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -1,6 +1,5 @@ import os -import fs.path from mock import patch from nose.tools import eq_ from moban.plugins import ENGINES @@ -8,6 +7,8 @@ from moban.jinja2.engine import Engine from moban.data_loaders.yaml import open_yaml +import fs.path + MODULE = "moban.plugins.template" diff --git a/tests/test_utils.py b/tests/test_utils.py index 535db5d2..4e999af4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,7 +3,6 @@ import stat from shutil import rmtree -import fs.path from mock import patch from nose import SkipTest from nose.tools import eq_, raises @@ -16,6 +15,8 @@ ) from moban.exceptions import FileNotFound +import fs.path + def create_file(test_file, permission): with open(test_file, "w") as f: diff --git a/tests/test_yaml_loader.py b/tests/test_yaml_loader.py index c8f2c8b1..ce453c8b 100644 --- a/tests/test_yaml_loader.py +++ b/tests/test_yaml_loader.py @@ -1,8 +1,9 @@ -import fs.path from nose.tools import eq_, raises from moban.data_loaders.yaml import open_yaml from moban.data_loaders.manager import load_data +import fs.path + def test_simple_yaml(): test_file = fs.path.join("tests", "fixtures", "simple.yaml") From 4462394b5fa4a436819695846f7c8037b6e6b9c5 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 20:55:12 +0100 Subject: [PATCH 074/167] :sparkles: use EnhancedOSFS for custom pypi and repo --- moban/fs_openers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/moban/fs_openers.py b/moban/fs_openers.py index a78b137e..ef792f56 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -5,7 +5,6 @@ import fs import fs.path -from fs.osfs import OSFS from fs.opener import Opener from fs.opener.registry import registry @@ -20,7 +19,7 @@ def open_fs(self, fs_url, parse_result, writeable, create, cwd): pypi_package_name, _, dir_path = parse_result.resource.partition("/") library_path = LIBRARIES.resource_path_of(pypi_package_name) root_path = fs.path.join(library_path, dir_path) - osfs = OSFS(root_path=root_path) + osfs = EnhancedOSFS(root_path=root_path) return osfs @@ -32,7 +31,7 @@ def open_fs(self, fs_url, parse_result, writeable, create, cwd): repo_name, _, dir_path = parse_result.resource.partition("/") actual_repo_path = fs.path.join(repo.get_moban_home(), repo_name) root_path = fs.path.join(actual_repo_path, dir_path) - osfs = OSFS(root_path=root_path) + osfs = EnhancedOSFS(root_path=root_path) return osfs From 7623738214fce0a7c5fbd4b282f9565c2f7b17a3 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 20:59:45 +0100 Subject: [PATCH 075/167] :shirt: fix linting errors --- moban/adapter/osfs.py | 5 ++--- moban/adapter/zipfs.py | 13 +++++-------- moban/fs_openers.py | 29 ++--------------------------- 3 files changed, 9 insertions(+), 38 deletions(-) diff --git a/moban/adapter/osfs.py b/moban/adapter/osfs.py index 77801327..36c75434 100644 --- a/moban/adapter/osfs.py +++ b/moban/adapter/osfs.py @@ -1,10 +1,9 @@ +import fs from fs.osfs import OSFS class EnhancedOSFS(OSFS): def geturl(self, path, purpose="download"): - # type: (Text, Text) -> Text if purpose != "download": - raise NoURL(path, purpose) - # file://D:\a\1\s\tests\fixtures\template can be not used by osfs itself. + raise fs.errors.NoURL(path, purpose) return "file://" + self.getsyspath(path).replace("\\", "/") diff --git a/moban/adapter/zipfs.py b/moban/adapter/zipfs.py index f1591bca..f792f628 100644 --- a/moban/adapter/zipfs.py +++ b/moban/adapter/zipfs.py @@ -11,15 +11,12 @@ def geturl(self, path, purpose="download"): class EnhancedZipFS(ZipFS): def __new__( cls, - file, # type: Union[Text, BinaryIO] - write=False, # type: bool - compression=zipfile.ZIP_DEFLATED, # type: int - encoding="utf-8", # type: Text - temp_fs="temp://__ziptemp__", # type: Text + file, + write=False, + compression=zipfile.ZIP_DEFLATED, + encoding="utf-8", + temp_fs="temp://__ziptemp__", ): - # type: (...) -> FS - # This magic returns a different class instance based on the - # value of the ``write`` parameter. if write: return WriteZipFS( file, diff --git a/moban/fs_openers.py b/moban/fs_openers.py index ef792f56..b67a5af0 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -8,8 +8,6 @@ from fs.opener import Opener from fs.opener.registry import registry -# FS_VERSION = _version_count(fs.__version__) - @registry.install class PypiFSOpener(Opener): @@ -42,14 +40,7 @@ class ZipOpener(Opener): protocols = ["zip"] - def open_fs( - self, - fs_url, # type: Text - parse_result, # type: ParseResult - writeable, # type: bool - create, # type: bool - cwd, # type: Text - ): + def open_fs(self, fs_url, parse_result, writeable, create, cwd): if not create and writeable: raise fs.errors.NotWriteable( "Unable to open existing ZIP file for writing" @@ -65,26 +56,10 @@ class OSFSOpener(Opener): protocols = ["file", "osfs"] - def open_fs( - self, - fs_url, # type: Text - parse_result, # type: ParseResult - writeable, # type: bool - create, # type: bool - cwd, # type: Text - ): - # type: (...) -> OSFS + def open_fs(self, fs_url, parse_result, writeable, create, cwd): from os.path import abspath, expanduser, normpath, join _path = abspath(join(cwd, expanduser(parse_result.resource))) path = normpath(_path) osfs = EnhancedOSFS(path, create=create) return osfs - - -def _version_count(version): - tokens = version.split(".") - sum = 0 - for index, token in enumerate(reversed(tokens)): - sum = sum + int(token) * (10 ** index) - return sum From a1863a198c0f63fd8f0c5b7bed8ab0e59f280ce8 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 21:22:33 +0100 Subject: [PATCH 076/167] :sparkles: enhance tar fs to have geturl function --- .../.moban.yml | 2 +- .../custom-config.tar | Bin 0 -> 2048 bytes .../custom-config/data.base.yaml | 2 - .../data.yml | 2 +- .../data2.yml | 2 +- moban/adapter/tarfs.py | 38 ++++++++++++++++++ moban/file_system.py | 11 +++++ moban/fs_openers.py | 22 ++++++++++ 8 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 docs/level-20-templates-configs-in-zip-or-tar/custom-config.tar delete mode 100644 docs/level-20-templates-configs-in-zip-or-tar/custom-config/data.base.yaml create mode 100644 moban/adapter/tarfs.py diff --git a/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml b/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml index 2593a187..f77cd19b 100644 --- a/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml +++ b/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml @@ -1,5 +1,5 @@ configuration: - configuration_dir: 'custom-config' + configuration_dir: 'tar://custom-config.tar' template_dir: - zip://templates.zip - cool-templates diff --git a/docs/level-20-templates-configs-in-zip-or-tar/custom-config.tar b/docs/level-20-templates-configs-in-zip-or-tar/custom-config.tar new file mode 100644 index 0000000000000000000000000000000000000000..b82f8705dd170aa9a94960f59c5dc153b16c116a GIT binary patch literal 2048 zcmeHDy9&cF4D{Tu&=(lFenJ06&;&bmQ>fFSe_s&~9=tY?L0Y6c32}FK+DPYBZ}iq* zdKPs+A!2Y26!#eX-tpQnQ(V@8vED=z&_@SooW-aB5708a{8YBi(iW-@8n&I_vw7}t pn3|Jj4iSd^Nr^QldflL=yyWSUrcydOsbrBQ2}lBxfF$sT1b+W2BKiOT literal 0 HcmV?d00001 diff --git a/docs/level-20-templates-configs-in-zip-or-tar/custom-config/data.base.yaml b/docs/level-20-templates-configs-in-zip-or-tar/custom-config/data.base.yaml deleted file mode 100644 index 3a35fadb..00000000 --- a/docs/level-20-templates-configs-in-zip-or-tar/custom-config/data.base.yaml +++ /dev/null @@ -1,2 +0,0 @@ -nihao: shijie -hello: shijie diff --git a/docs/level-20-templates-configs-in-zip-or-tar/data.yml b/docs/level-20-templates-configs-in-zip-or-tar/data.yml index 168bfb12..3d225095 100644 --- a/docs/level-20-templates-configs-in-zip-or-tar/data.yml +++ b/docs/level-20-templates-configs-in-zip-or-tar/data.yml @@ -1,2 +1,2 @@ -overrides: data.base.yaml +overrides: data.in.tar.yaml hello: world diff --git a/docs/level-20-templates-configs-in-zip-or-tar/data2.yml b/docs/level-20-templates-configs-in-zip-or-tar/data2.yml index e75891e6..58a044c2 100644 --- a/docs/level-20-templates-configs-in-zip-or-tar/data2.yml +++ b/docs/level-20-templates-configs-in-zip-or-tar/data2.yml @@ -1,2 +1,2 @@ -overrides: data.base.yaml +overrides: data.in.tar.yaml hello: world2 diff --git a/moban/adapter/tarfs.py b/moban/adapter/tarfs.py new file mode 100644 index 00000000..c3d13340 --- /dev/null +++ b/moban/adapter/tarfs.py @@ -0,0 +1,38 @@ +import six +import os +from fs.tarfs import TarFS, ReadTarFS, WriteTarFS + + +class EnhancedReadTarFS(ReadTarFS): + def geturl(self, path, purpose="download"): + return "tar://%s/%s" % (self._file, path) + + +class EnhancedTarFS(TarFS): + def __new__( + cls, + file, + write=False, + compression=None, + encoding="utf-8", + temp_fs="temp://__tartemp__", + ): + if isinstance(file, (six.text_type, six.binary_type)): + file = os.path.expanduser(file) + filename = file + else: + filename = getattr(file, "name", "") + + if write and compression is None: + compression = None + for comp, extensions in six.iteritems(cls._compression_formats): + if filename.endswith(extensions): + compression = comp + break + + if write: + return WriteTarFS( + file, compression=compression, encoding=encoding, temp_fs=temp_fs + ) + else: + return EnhancedReadTarFS(file, encoding=encoding) diff --git a/moban/file_system.py b/moban/file_system.py index 7787956f..0e080d73 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -88,6 +88,11 @@ def is_dir(path): with fs.open_fs(zip_file + ".zip") as the_fs: return the_fs.isdir(folder) + if "tar://" in path: + zip_file, folder = path.split(".tar/") + with fs.open_fs(zip_file + ".tar") as the_fs: + return the_fs.isdir(folder) + path = to_unicode(path) dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) @@ -113,6 +118,12 @@ def exists(path): return True except fs.errors.CreateFailed: return False + if "tar://" in path: + try: + with fs.open_fs(path) as the_fs: + return True + except fs.errors.CreateFailed: + return False dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) diff --git a/moban/fs_openers.py b/moban/fs_openers.py index b67a5af0..5e23d3a5 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -1,6 +1,7 @@ from moban import repo from moban.adapter.osfs import EnhancedOSFS from moban.adapter.zipfs import EnhancedZipFS +from moban.adapter.tarfs import EnhancedTarFS from moban.plugins.library import LIBRARIES import fs @@ -49,6 +50,27 @@ def open_fs(self, fs_url, parse_result, writeable, create, cwd): return zip_fs +@registry.install +class TarOpener(Opener): + """`TarFS` opener. + """ + + protocols = ["tar"] + + def open_fs( + self, + fs_url, + parse_result, + writeable, + create, + cwd, + ): + if not create and writeable: + raise fs.errors.NotWriteable("Unable to open existing TAR file for writing") + tar_fs = EnhancedTarFS(parse_result.resource, write=create) + return tar_fs + + @registry.install class OSFSOpener(Opener): """`OSFS` opener. From 9f0c3ca9c261bb878abdfd6a5d970a1c524e69f1 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 21:22:50 +0100 Subject: [PATCH 077/167] :shirt: update coding style --- moban/adapter/tarfs.py | 8 ++++++-- moban/fs_openers.py | 15 +++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/moban/adapter/tarfs.py b/moban/adapter/tarfs.py index c3d13340..87f391e4 100644 --- a/moban/adapter/tarfs.py +++ b/moban/adapter/tarfs.py @@ -1,5 +1,6 @@ -import six import os + +import six from fs.tarfs import TarFS, ReadTarFS, WriteTarFS @@ -32,7 +33,10 @@ def __new__( if write: return WriteTarFS( - file, compression=compression, encoding=encoding, temp_fs=temp_fs + file, + compression=compression, + encoding=encoding, + temp_fs=temp_fs, ) else: return EnhancedReadTarFS(file, encoding=encoding) diff --git a/moban/fs_openers.py b/moban/fs_openers.py index 5e23d3a5..d4cc484b 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -1,7 +1,7 @@ from moban import repo from moban.adapter.osfs import EnhancedOSFS -from moban.adapter.zipfs import EnhancedZipFS from moban.adapter.tarfs import EnhancedTarFS +from moban.adapter.zipfs import EnhancedZipFS from moban.plugins.library import LIBRARIES import fs @@ -57,16 +57,11 @@ class TarOpener(Opener): protocols = ["tar"] - def open_fs( - self, - fs_url, - parse_result, - writeable, - create, - cwd, - ): + def open_fs(self, fs_url, parse_result, writeable, create, cwd): if not create and writeable: - raise fs.errors.NotWriteable("Unable to open existing TAR file for writing") + raise fs.errors.NotWriteable( + "Unable to open existing TAR file for writing" + ) tar_fs = EnhancedTarFS(parse_result.resource, write=create) return tar_fs From 36cf3fdab3ef8a84ea20420dff162265a5151db4 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 21:26:07 +0100 Subject: [PATCH 078/167] :books: level 20 will provide the data in an alien file system. resolves #304 --- docs/level-20-templates-configs-in-zip-or-tar/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/level-20-templates-configs-in-zip-or-tar/README.rst b/docs/level-20-templates-configs-in-zip-or-tar/README.rst index 6858a6ef..45a1d2e1 100644 --- a/docs/level-20-templates-configs-in-zip-or-tar/README.rst +++ b/docs/level-20-templates-configs-in-zip-or-tar/README.rst @@ -5,7 +5,7 @@ On top of level 6, you could have files in a zip or tar. In the following example:: configuration: - configuration_dir: 'custom-config' + configuration_dir: 'tar://custom-config.tar' template_dir: - zip://templates.zip - cool-templates From 61ee49ea26efe8af3477187187eca9aace06c7dc Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 22:16:37 +0100 Subject: [PATCH 079/167] :books: write files to an alien file system. resolves #306 --- .../.moban.yml | 4 +- .../README.rst | 8 ++-- .../a.output | 9 ---- moban/file_system.py | 43 ++++++++++++++----- moban/utils.py | 2 + tests/test_docs.py | 15 ++++++- 6 files changed, 54 insertions(+), 27 deletions(-) delete mode 100644 docs/level-20-templates-configs-in-zip-or-tar/a.output diff --git a/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml b/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml index f77cd19b..8e124971 100644 --- a/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml +++ b/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml @@ -5,9 +5,9 @@ configuration: - cool-templates - '.' targets: - - output: a.output + - output: 'tar://a.tar/a.output' configuration: data.yml template: template.in.zip.jj2 - - output: a.output2 + - output: 'zip://a.zip/a.output2' configuration: data2.yml template: subfolder/template.in.zip.jj2 diff --git a/docs/level-20-templates-configs-in-zip-or-tar/README.rst b/docs/level-20-templates-configs-in-zip-or-tar/README.rst index 45a1d2e1..16977c2e 100644 --- a/docs/level-20-templates-configs-in-zip-or-tar/README.rst +++ b/docs/level-20-templates-configs-in-zip-or-tar/README.rst @@ -11,10 +11,10 @@ In the following example:: - cool-templates - '.' targets: - - output: a.output + - output: 'tar://a.tar/a.output' configuration: data.yml template: template.in.zip.jj2 - - output: a.output2 + - output: 'zip://a.zip/a.output2' configuration: data2.yml template: subfolder/template.in.zip.jj2 @@ -32,7 +32,7 @@ Here is the command to launch it: moban -'a.output' is the generated file:: +'a.output' is the generated file in a.tar:: ========header============ @@ -44,7 +44,7 @@ Here is the command to launch it: ========footer============ -`a.output2` is:: +`a.output2` is in a.zip:: ========header============ diff --git a/docs/level-20-templates-configs-in-zip-or-tar/a.output b/docs/level-20-templates-configs-in-zip-or-tar/a.output deleted file mode 100644 index 9649d143..00000000 --- a/docs/level-20-templates-configs-in-zip-or-tar/a.output +++ /dev/null @@ -1,9 +0,0 @@ -========header============ - -world - -shijie - -this demonstrates jinja2's include statement - -========footer============ diff --git a/moban/file_system.py b/moban/file_system.py index 0e080d73..746e15d8 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -55,12 +55,23 @@ def open_fs(path): @log_fs_failure def read_unicode(path): - path = to_unicode(path) - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - with fs.open_fs(dir_name) as fs_system: - with fs_system.open(the_file_name) as file_handle: - return file_handle.read() + if "zip://" in path: + zip_file, folder = path.split(".zip/") + with fs.open_fs(zip_file + ".zip") as the_fs: + with the_fs.open(folder) as file_handle: + return file_handle.read() + elif "tar://" in path: + tar_file, folder = path.split(".tar/") + with fs.open_fs(tar_file + ".tar") as the_fs: + with the_fs.open(folder) as file_handle: + return file_handle.read() + else: + path = to_unicode(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as fs_system: + with fs_system.open(the_file_name) as file_handle: + return file_handle.read() @log_fs_failure @@ -74,11 +85,21 @@ def read_binary(path): @log_fs_failure def write_bytes(filename, bytes_content): - filename = to_unicode(filename) - dir_name = fs.path.dirname(filename) - the_file_name = fs.path.basename(filename) - with fs.open_fs(dir_name) as the_fs: - the_fs.writebytes(the_file_name, bytes_content) + if "zip://" in filename: + zip_file, folder = filename.split(".zip/") + with fs.open_fs(zip_file + ".zip", create=True) as the_fs: + the_fs.writebytes(folder, bytes_content) + + elif "tar://" in filename: + tar_file, folder = filename.split(".tar/") + with fs.open_fs(tar_file + ".tar", create=True) as the_fs: + the_fs.writebytes(folder, bytes_content) + else: + filename = to_unicode(filename) + dir_name = fs.path.dirname(filename) + the_file_name = fs.path.basename(filename) + with fs.open_fs(dir_name) as the_fs: + the_fs.writebytes(the_file_name, bytes_content) @log_fs_failure diff --git a/moban/utils.py b/moban/utils.py index 128ad98b..8d90cc91 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -63,6 +63,8 @@ def file_permissions_copy(source, dest): def file_permissions(afile): + if "zip://" in afile or "tar://" in afile: + return 755 if not os.path.exists(afile): raise exceptions.FileNotFound(afile) if sys.platform == "win32": diff --git a/tests/test_docs.py b/tests/test_docs.py index 526de056..49fac476 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -4,6 +4,7 @@ from mock import patch from moban.main import main +from moban import file_system from nose.tools import eq_ @@ -115,7 +116,7 @@ def test_level_20(self): """ ) folder = "level-20-templates-configs-in-zip-or-tar" - self._raw_moban(["moban"], folder, expected, "a.output2") + self._raw_moban_with_fs(["moban"], folder, expected, "zip://a.zip/a.output2") def test_level_7(self): expected = custom_dedent( @@ -306,6 +307,13 @@ def _raw_moban(self, args, folder, expected, output): _verify_content(output, expected) os.unlink(output) + def _raw_moban_with_fs(self, args, folder, expected, output): + os.chdir(os.path.join("docs", folder)) + with patch.object(sys, "argv", args): + main() + _verify_content_with_fs(output, expected) + os.unlink(output.split('/')[2]) # fixme later + def tearDown(self): if os.path.exists(".moban.hashes"): os.unlink(".moban.hashes") @@ -316,3 +324,8 @@ def _verify_content(file_name, expected): with open(file_name, "r") as f: content = f.read() eq_(content, expected) + + +def _verify_content_with_fs(file_name, expected): + content = file_system.read_unicode(file_name) + eq_(content, expected) From efa9eb11efc4b3fa5a3001f108f84e389d763aec Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 22:16:53 +0100 Subject: [PATCH 080/167] :shirt: update coding style --- tests/test_docs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index 49fac476..4de77109 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -3,8 +3,8 @@ from textwrap import dedent from mock import patch -from moban.main import main from moban import file_system +from moban.main import main from nose.tools import eq_ @@ -116,7 +116,9 @@ def test_level_20(self): """ ) folder = "level-20-templates-configs-in-zip-or-tar" - self._raw_moban_with_fs(["moban"], folder, expected, "zip://a.zip/a.output2") + self._raw_moban_with_fs( + ["moban"], folder, expected, "zip://a.zip/a.output2" + ) def test_level_7(self): expected = custom_dedent( @@ -312,7 +314,7 @@ def _raw_moban_with_fs(self, args, folder, expected, output): with patch.object(sys, "argv", args): main() _verify_content_with_fs(output, expected) - os.unlink(output.split('/')[2]) # fixme later + os.unlink(output.split("/")[2]) # fixme later def tearDown(self): if os.path.exists(".moban.hashes"): From 44a1d752852c35534b71a3d53b7cbf857daa70c6 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 22:23:40 +0100 Subject: [PATCH 081/167] :bug: avoid create a zip folder --- moban/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 8d90cc91..9646abd8 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -75,9 +75,12 @@ def file_permissions(afile): def write_file_out(filename, content): if PY2 and content.__class__.__name__ == "unicode": content = content.encode("utf-8") - dest_folder = os.path.dirname(filename) - if dest_folder: - mkdir_p(dest_folder) + + if "zip://" not in filename or "tar://" not in filename: + # fix me + dest_folder = os.path.dirname(filename) + if dest_folder: + mkdir_p(dest_folder) file_system.write_bytes(filename, content) From 374c2e598ab147d01b19c0e799bace69f90f5c70 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 22:25:12 +0100 Subject: [PATCH 082/167] :bug: unicode bug again in python 2 --- moban/file_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/file_system.py b/moban/file_system.py index 746e15d8..db3255c3 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -55,6 +55,7 @@ def open_fs(path): @log_fs_failure def read_unicode(path): + path = to_unicode(path) if "zip://" in path: zip_file, folder = path.split(".zip/") with fs.open_fs(zip_file + ".zip") as the_fs: @@ -66,7 +67,6 @@ def read_unicode(path): with the_fs.open(folder) as file_handle: return file_handle.read() else: - path = to_unicode(path) dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) with fs.open_fs(dir_name) as fs_system: From a38e14124683fbd9542352160e0626f63b9845f9 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 28 Jul 2019 22:26:32 +0100 Subject: [PATCH 083/167] :bug: inverse the logic for making windows dirs --- moban/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/utils.py b/moban/utils.py index 9646abd8..80ba47ea 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -76,7 +76,7 @@ def write_file_out(filename, content): if PY2 and content.__class__.__name__ == "unicode": content = content.encode("utf-8") - if "zip://" not in filename or "tar://" not in filename: + if "zip://" not in filename and "tar://" not in filename: # fix me dest_folder = os.path.dirname(filename) if dest_folder: From d56d2e61fc5da76b1e87bd58d4398f497e30edfb Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 29 Jul 2019 22:24:15 +0100 Subject: [PATCH 084/167] :books: copy from zip to zip file. resolves #307. :sparkles: write multiple files into a zip. fix #308" --- docs/README.rst | 6 +- docs/index.rst | 3 +- .../.moban.yml | 19 ++++ .../README.rst | 83 ++++++++++++++++++ .../template-sources.tar | Bin 0 -> 10752 bytes .../template-sources.zip | Bin 0 -> 2574 bytes moban/file_system.py | 58 +++++++++--- moban/mobanfile/__init__.py | 1 + moban/mobanfile/targets.py | 1 + moban/mobanfile/templates.py | 70 +++++++++++++-- moban/plugins/template.py | 15 +++- moban/utils.py | 6 +- tests/test_docs.py | 30 +++++++ 13 files changed, 262 insertions(+), 30 deletions(-) create mode 100644 docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml create mode 100644 docs/level-21-copy-templates-into-an-alien-file-system/README.rst create mode 100644 docs/level-21-copy-templates-into-an-alien-file-system/template-sources.tar create mode 100644 docs/level-21-copy-templates-into-an-alien-file-system/template-sources.zip diff --git a/docs/README.rst b/docs/README.rst index b4aff488..cfbe4a9f 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -22,7 +22,9 @@ This section covers the use cases for moban. Please check them out individually. #. `Force template type from moban file`_ #. `User defined template types`_ #. `Select a group target to run`_ - +#. `Template files in a zip or tar`_ +#. `Template copying from a zip to a zip`_ + .. _Jinja2 command line: level-1-jinja2-cli .. _Template inheritance: level-2-template-inheritance .. _Data override: level-3-data-override @@ -42,3 +44,5 @@ This section covers the use cases for moban. Please check them out individually. .. _Force template type from moban file: level-17-force-template-type-from-moban-file .. _User defined template types: level-18-user-defined-template-types .. _Select a group target to run: level-19-moban-a-sub-group-in-targets +.. _Template files in a zip or tar: level-20-templates-configs-in-zip-or-tar +.. _Template copying from a zip to a zip: level-21-copy-templates-into-an-alien-file-system diff --git a/docs/index.rst b/docs/index.rst index ace1fb1b..d5a85674 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,7 +34,8 @@ examples folder. level-17-force-template-type-from-moban-file/README.rst level-18-user-defined-template-types/README.rst level-19-moban-a-sub-group-in-targets/README.rst - + level-20-templates-configs-in-zip-or-tar/README.rst + level-21-copy-templates-into-an-alien-file-system/README.rst For more complex use case, please look at `its usage in pyexcel project `_ diff --git a/docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml b/docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml new file mode 100644 index 00000000..aefbad15 --- /dev/null +++ b/docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml @@ -0,0 +1,19 @@ +configuration: + template_dir: + - "zip://template-sources.zip" +targets: + - output: "zip://my.zip/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip/target_without_template_type" + template: file_extension_will_trigger.copy + - "zip://my.zip/target_in_short_form": as_long_as_this_one_has.copy + - output: "zip://my.zip/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip/test-dir" + template: dir-for-copying + template_type: copy + - output: "zip://my.zip/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy diff --git a/docs/level-21-copy-templates-into-an-alien-file-system/README.rst b/docs/level-21-copy-templates-into-an-alien-file-system/README.rst new file mode 100644 index 00000000..684367a8 --- /dev/null +++ b/docs/level-21-copy-templates-into-an-alien-file-system/README.rst @@ -0,0 +1,83 @@ +Level 21: template copying from a zip to a zip +================================================================================ + +In level 15, with `.moban.yml`, you can copy templates to your destination. Now +with similiar moban syntax, let me show how to create a new zip file where +all templates are copied to. + +Explicit syntax:: + + targets: + - output: "zip://your.zip/explicit" + template: template_file + template_type: copy + + +Implicit syntax:: + + targets: + - output: "zip://your.zip/implicit" + template: template_file.copy + + +Shorthand syntax:: + + targets: + - "zip://your.zip/shorthand": template_file.copy + + +No implicit nor short hand syntax for the following directory copying unless +you take a look at `force-template-type`. When you read +`level-17-force-template-type-from-moban-file/README.rst`, you will find +out more. + + +Directory copying syntax:: + + + targets: + - output: "zip://your.zip/dest-dir" + template: source-dir + template_type: copy + + +Recursive directory copying syntax:: + + + targets: + - output: "zip://your.zip/dest-dir" + template: source-dir/** + template_type: copy + + +Evaluation +-------------------------------------------------------------------------------- + +Here is example moban file for copying:: + + configuration: + template_dir: + - "zip://template-sources.zip" + targets: + - output: "zip://my.zip/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip/target_without_template_type" + template: file_extension_will_trigger.copy + - "zip://my.zip/target_in_short_form": as_long_as_this_one_has.copy + - output: "zip://my.zip/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip/test-dir" + template: dir-for-copying + template_type: copy + - output: "zip://my.zip/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy + + +template copy does: + + +#. copies any template inside pre-declared template directory to anywhere. moban will create directory if needed. +#. copies any directory to anywhere. If "**" is followed, moban attempts to do recursive copying. diff --git a/docs/level-21-copy-templates-into-an-alien-file-system/template-sources.tar b/docs/level-21-copy-templates-into-an-alien-file-system/template-sources.tar new file mode 100644 index 0000000000000000000000000000000000000000..2f13227fd560f0e893e00691f01d6332def4f280 GIT binary patch literal 10752 zcmeHNOOM(x5a!%p;TzHlg5x}pdac?^5A8pQ3>kuVLyT;PUH*N?z!I^n5*|sq-8u)H z#Ls*)-^_T7HyLz(QA(##LnC0_AvRkqYU&T)VLuWtQq%H7Re6(CsPRIov4VN2#RkT6x!#H*Yfmu{k1ORL z2R?K1??X|%$wK5X7RA`WYMpZfV=YTH2aQURr=UFZV(wfB%0CYGe1D(+K0D<jv8S*e3onKR)HZ zKSc}R_}>ot^`i(wgeJd!5+liyZ-8*3mr|ikkZLs|v(gZ=gq?-3O=dubs2Bp$A-3ar zzVBq;ZHEM23=~?(VnnngT-zQ-i7W#}zR3bB%@3)bxk|qmnJ8ei8_agxJ$lXt)eEbG z+A98QJ?uNK_WJlQ3}YAnAMxw*W*`40N9HEaBF~cpXIZ?d|6R)c zb!;2@Rd$@4#uV4ti!xBuL#L<_sV3aCc@vp4+r(TdZy&03gZwcF@yM`Ts}2{Ltn9 o`Vjvh#q-~4yZ05(ZS3KH^=9($;M_qkl5mqa1RMemfqoJA14BUqTL1t6 literal 0 HcmV?d00001 diff --git a/docs/level-21-copy-templates-into-an-alien-file-system/template-sources.zip b/docs/level-21-copy-templates-into-an-alien-file-system/template-sources.zip new file mode 100644 index 0000000000000000000000000000000000000000..5085110c10f6301afc5ac8775a078adb20101dfc GIT binary patch literal 2574 zcmbVMTSydP7@c)@rBq_0BrTD_BrJAjF}i3LD%s7&ZOsB-WSX7*cb%M_*=Asm`=SFFF7 zusP;z1mRcXjGsKf#cG>AA=lH&}oFsX8uz2P2LB<1?HogJ)%w`y1&{{t0392-**qo}N$x&RF& zia0(5jaiA|Jw(xihsWKmZS=V3-sq8<*B+|Xx9`GC4}a3OXbg$V`xAK1cx?o#SZ#_I zDHTPc0;?iQkzz836d1PWSX66Y!3k0;w!UvGNjj~04n+uY6#^;3QG6K7ifOuahTw2KZ8 z+s5V|inWrpiO!5_isan3c!fq%$=) z985NI)Ee4&0iTy`BgQO6p4fXJhUK*k zQ8OS-0b_9Lftm{lix@RS&|1li72jQk(K_s!#+hs~pPW|Jm~ndEz9p*a5-)$!ZUgBD zOO(&Dp&{*ZLo{ZczM*xFY?;guEt4-dLdJB{6M6)Mtg~)86lWoJLNT1rp-!C>zx@L( CwxeDE literal 0 HcmV?d00001 diff --git a/moban/file_system.py b/moban/file_system.py index db3255c3..53890f97 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -77,10 +77,19 @@ def read_unicode(path): @log_fs_failure def read_binary(path): path = to_unicode(path) - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - with fs.open_fs(dir_name) as fs_system: - return fs_system.readbytes(the_file_name) + if "zip://" in path: + zip_file, folder = path.split(".zip/") + with fs.open_fs(zip_file + ".zip") as the_fs: + return the_fs.readbytes(folder) + elif "tar://" in path: + tar_file, folder = path.split(".tar/") + with fs.open_fs(tar_file + ".tar") as the_fs: + return the_fs.readbytes(folder) + else: + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as fs_system: + return fs_system.readbytes(the_file_name) @log_fs_failure @@ -133,18 +142,39 @@ def is_file(path): @log_fs_failure def exists(path): path = to_unicode(path) + if "zip://" in path: - try: - with fs.open_fs(path) as the_fs: - return True - except fs.errors.CreateFailed: - return False + if path.endswith(".zip"): + zip_file, folder = path, "/" + try: + with fs.open_fs(zip_file) as the_fs: + return True + except fs.errors.CreateFailed: + return False + else: + zip_file, folder = path.split(".zip/") + try: + with fs.open_fs(zip_file + ".zip") as the_fs: + return the_fs.exists(folder) + except fs.errors.CreateFailed: + return False + if "tar://" in path: - try: - with fs.open_fs(path) as the_fs: - return True - except fs.errors.CreateFailed: - return False + if path.endswith(".tar"): + zip_file, folder = path, "/" + try: + with fs.open_fs(zip_file) as the_fs: + return True + except fs.errors.CreateFailed: + return False + else: + zip_file, folder = path.split(".tar/") + try: + with fs.open_fs(zip_file + ".tar") as the_fs: + return the_fs.exists(folder) + except fs.errors.CreateFailed: + return False + dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index 98432afd..09d56aa2 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -109,6 +109,7 @@ def _iterate_list_of_dicts(list_of_dict): def handle_targets(merged_options, targets): list_of_templating_parameters = parse_targets(merged_options, targets) + jobs_for_each_engine = OrderedDict() for target in list_of_templating_parameters: diff --git a/moban/mobanfile/targets.py b/moban/mobanfile/targets.py index e32c8767..07505df0 100644 --- a/moban/mobanfile/targets.py +++ b/moban/mobanfile/targets.py @@ -40,6 +40,7 @@ def parse_targets(options, targets): def _handle_explicit_target(options, target): + common_data_file = options[constants.LABEL_CONFIG] default_template_type = options[constants.LABEL_TEMPLATE_TYPE] template_file = target.get( diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 8c8fccd9..079f13e9 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -8,14 +8,21 @@ def handle_template(template_file, output, template_dirs): log.info("handling %s" % template_file) + if template_file.endswith("**"): source_dir = template_file[:-3] src_path = find_file_in_template_dirs(source_dir, template_dirs) if src_path: - for a_triple in _listing_directory_files_recusively( - source_dir, src_path, output - ): - yield a_triple + if "zip://" in src_path: + for a_triple in _listing_zip_directory_files_recusively( + source_dir, src_path, output + ): + yield a_triple + else: + for a_triple in _listing_directory_files_recusively( + source_dir, src_path, output + ): + yield a_triple else: reporter.report_error_message( "{0} cannot be found".format(template_file) @@ -30,10 +37,16 @@ def handle_template(template_file, output, template_dirs): "{0} cannot be found".format(template_file) ) elif file_system.is_dir(template_file_on_disk): - for a_triple in _list_dir_files( - template_file, template_file_on_disk, output - ): - yield a_triple + if "zip://" in template_file_on_disk: + for a_triple in _list_zip_dir_files( + template_file, template_file_on_disk, output + ): + yield a_triple + else: + for a_triple in _list_dir_files( + template_file, template_file_on_disk, output + ): + yield a_triple else: template_type = _get_template_type(template_file) yield (template_file, output, template_type) @@ -54,6 +67,24 @@ def _list_dir_files(source, actual_source_path, dest): yield (src_file_under_dir, dest_file_under_dir, template_type) +def _list_zip_dir_files(source, actual_source_path, dest): + zip_file, folder = actual_source_path.split(".zip/") + with file_system.open_fs(zip_file + ".zip") as fs_handle: + for file_name in fs_handle.listdir(file_system.to_unicode(folder)): + if fs_handle.isfile( + file_system.to_unicode(folder + "/" + file_name) + ): + # please note jinja2 does NOT like windows path + # hence the following statement looks like cross platform + # src_file_under_dir = os.path.join(source, file_name) + # but actually it breaks windows instead. + src_file_under_dir = "%s/%s" % (source, file_name) + + dest_file_under_dir = dest + "/" + file_name + template_type = _get_template_type(src_file_under_dir) + yield (src_file_under_dir, dest_file_under_dir, template_type) + + def _listing_directory_files_recusively(source, actual_source_path, dest): with file_system.open_fs(actual_source_path) as fs_handle: for file_name in fs_handle.listdir(u"."): @@ -70,6 +101,29 @@ def _listing_directory_files_recusively(source, actual_source_path, dest): yield a_triple +def _listing_zip_directory_files_recusively(source, actual_source_path, dest): + zip_file, folder = actual_source_path.split(".zip/") + print(actual_source_path) + + with file_system.open_fs(zip_file + ".zip") as fs_handle: + for file_name in fs_handle.listdir(file_system.to_unicode(folder)): + src_file_under_dir = source + "/" + file_name + dest_file_under_dir = dest + "/" + file_name + real_src_file = "%s/%s" % (actual_source_path, file_name) + if fs_handle.isfile( + file_system.to_unicode(folder + "/" + file_name) + ): + template_type = _get_template_type(src_file_under_dir) + yield (src_file_under_dir, dest_file_under_dir, template_type) + elif fs_handle.isdir( + file_system.to_unicode(folder + "/" + file_name) + ): + for a_triple in _listing_zip_directory_files_recusively( + src_file_under_dir, real_src_file, dest_file_under_dir + ): + yield a_triple + + def _get_template_type(template_file): _, extension = file_system.path_splitext(template_file) if extension: diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 882e2927..8ba1b398 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -6,6 +6,7 @@ from lml.plugin import PluginManager from moban.hashstore import HASH_STORE from moban.deprecated import deprecated +from moban.buffered_writer import BufferedWriter from moban.plugins.context import Context from moban.plugins.library import LIBRARIES from moban.plugins.strategy import Strategy @@ -68,6 +69,7 @@ def __init__(self, template_dirs, context_dirs, engine): self.engine = engine self.templated_count = 0 self.file_count = 0 + self.buffered_writer = BufferedWriter() def report(self): if self.templated_count == 0: @@ -96,6 +98,7 @@ def render_to_file(self, template_file, data_file, output_file): if flag: reporter.report_templating(template_file, output_file) self.templated_count += 1 + self.buffered_writer.close() def render_string_to_file( self, template_in_string, data_file, output_file @@ -110,6 +113,7 @@ def render_string_to_file( if flag: reporter.report_templating(template_abs_path, output_file) self.templated_count += 1 + self.buffered_writer.close() def apply_template(self, template_abs_path, template, data, output_file): rendered_content = self.engine.apply_template( @@ -124,21 +128,26 @@ def apply_template(self, template_abs_path, template, data, output_file): output_file, rendered_content, template_abs_path ) if flag: - utils.write_file_out(output_file, rendered_content) - utils.file_permissions_copy(template_abs_path, output_file) + self.buffered_writer.write_file_out( + output_file, rendered_content + ) + if "zip://" not in output_file and "tar://" not in output_file: + utils.file_permissions_copy(template_abs_path, output_file) return flag except exceptions.FileNotFound: - utils.write_file_out(output_file, rendered_content) + self.buffered_writer.write_file_out(output_file, rendered_content) return True def render_to_files(self, array_of_template_targets): sta = Strategy(array_of_template_targets) + sta.process() choice = sta.what_to_do() if choice == Strategy.DATA_FIRST: self._render_with_finding_data_first(sta.data_file_index) else: self._render_with_finding_template_first(sta.template_file_index) + self.buffered_writer.close() def _render_with_finding_template_first(self, template_file_index): for (template_file, data_output_pairs) in template_file_index.items(): diff --git a/moban/utils.py b/moban/utils.py index 80ba47ea..07641189 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -127,6 +127,7 @@ def verify_the_existence_of_directories(dirs): results = [] for directory in dirs: + if file_system.exists(directory): results.append(directory) continue @@ -147,8 +148,7 @@ def verify_the_existence_of_directories(dirs): def find_file_in_template_dirs(src, template_dirs): LOG.debug(template_dirs) for folder in template_dirs: - with file_system.open_fs(folder) as fs_handle: - if fs_handle.exists(file_system.to_unicode(src)): - return fs_handle.geturl(src) + if file_system.exists(folder + "/" + src): + return folder + "/" + src else: return None diff --git a/tests/test_docs.py b/tests/test_docs.py index 4de77109..60ee3a65 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -252,6 +252,27 @@ def test_level_15_copy_templates_as_target(self): ), ) + def test_level_21_copy_templates_into_zips(self): + expected = "test file\n" + + folder = "level-21-copy-templates-into-an-alien-file-system" + criterias = [ + ["zip://my.zip/simple.file", expected], + [ + "zip://my.zip/target_without_template_type", + "file extension will trigger copy engine\n", + ], + [ + "zip://my.zip/target_in_short_form", + ( + "it is OK to have a short form, " + + "but the file to be 'copied' shall have 'copy' extension, " + + "so as to trigger ContentForwardEngine, 'copy' engine.\n" + ), + ], + ] + self._raw_moban_with_fs2(["moban"], folder, criterias) + def test_level_16_group_targets_using_template_type(self): expected = "test file\n" @@ -316,6 +337,15 @@ def _raw_moban_with_fs(self, args, folder, expected, output): _verify_content_with_fs(output, expected) os.unlink(output.split("/")[2]) # fixme later + def _raw_moban_with_fs2(self, args, folder, criterias): + os.chdir(os.path.join("docs", folder)) + with patch.object(sys, "argv", args): + main() + + for output, expected in criterias: + _verify_content_with_fs(output, expected) + os.unlink(output.split("/")[2]) # fixme later + def tearDown(self): if os.path.exists(".moban.hashes"): os.unlink(".moban.hashes") From 9ba3c840514e765acac2542ee3faf47671824918 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 29 Jul 2019 23:10:41 +0100 Subject: [PATCH 085/167] :newspaper: add missing source file --- moban/buffered_writer.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 moban/buffered_writer.py diff --git a/moban/buffered_writer.py b/moban/buffered_writer.py new file mode 100644 index 00000000..e000dce9 --- /dev/null +++ b/moban/buffered_writer.py @@ -0,0 +1,33 @@ +from moban import utils, file_system + +import fs +import fs.path + + +class BufferedWriter(object): + def __init__(self): + self.fs_list = {} + + def write_file_out(self, filename, content): + if "zip://" in filename: + self.write_file_out_to_zip(filename, content) + else: + utils.write_file_out(filename, content) + + def write_file_out_to_zip(self, filename, content): + zip_file, file_name = filename.split(".zip/") + zip_file = zip_file + ".zip" + if zip_file not in self.fs_list: + self.fs_list[zip_file] = fs.open_fs( + file_system.to_unicode(zip_file), create=True + ) + base_dirs = fs.path.dirname(file_name) + if not self.fs_list[zip_file].exists(base_dirs): + self.fs_list[zip_file].makedirs(base_dirs) + self.fs_list[zip_file].writebytes( + file_system.to_unicode(file_name), content + ) + + def close(self): + for fsx in self.fs_list.values(): + fsx.close() From 1c1d5bff91a40d20d270f3104215a34fae222a7d Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 29 Jul 2019 23:21:51 +0100 Subject: [PATCH 086/167] :ambulance: python 2 non-unicode string caught again by pyfilesystem2. related to #303 --- moban/file_system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index 53890f97..aae89955 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -116,12 +116,12 @@ def is_dir(path): if "zip://" in path: zip_file, folder = path.split(".zip/") with fs.open_fs(zip_file + ".zip") as the_fs: - return the_fs.isdir(folder) + return the_fs.isdir(to_unicode(folder)) if "tar://" in path: zip_file, folder = path.split(".tar/") with fs.open_fs(zip_file + ".tar") as the_fs: - return the_fs.isdir(folder) + return the_fs.isdir(to_unicode(folder)) path = to_unicode(path) dir_name = fs.path.dirname(path) From a2155761eb3de80fa78f173286725055647d9d1f Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 30 Jul 2019 07:59:25 +0100 Subject: [PATCH 087/167] :bug: unicode appears in python2 and but we write bytes in python2 --- moban/plugins/template.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 8ba1b398..05db6540 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -119,9 +119,8 @@ def apply_template(self, template_abs_path, template, data, output_file): rendered_content = self.engine.apply_template( template, data, output_file ) - if PY3_ABOVE: - if not isinstance(rendered_content, bytes): - rendered_content = rendered_content.encode("utf-8") + if not isinstance(rendered_content, bytes): + rendered_content = rendered_content.encode("utf-8") try: flag = HASH_STORE.is_file_changed( From 8d816a6d7bef556bc402550f98102de160b8a371 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 31 Jul 2019 07:59:11 +0100 Subject: [PATCH 088/167] :microscope: initial unit test for moban file system layer --- moban/file_system.py | 32 +++++- .../fixtures/file_system/template-sources.tar | Bin 0 -> 10752 bytes .../fixtures/file_system/template-sources.zip | Bin 0 -> 2574 bytes tests/test_file_system.py | 106 ++++++++++++++++++ 4 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/file_system/template-sources.tar create mode 100644 tests/fixtures/file_system/template-sources.zip create mode 100644 tests/test_file_system.py diff --git a/moban/file_system.py b/moban/file_system.py index aae89955..3c5c43c5 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -31,10 +31,19 @@ def wrapper(*args, **kwds): @contextmanager def open_file(path): path = to_unicode(path) - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - the_fs = fs.open_fs(dir_name) - f = the_fs.open(the_file_name) + if "zip://" in path: + zip_file, folder = path.split(".zip/") + the_fs = fs.open_fs(zip_file + ".zip") + f = the_fs.open(folder) + elif "tar://" in path: + tar_file, folder = path.split(".tar/") + the_fs = fs.open_fs(tar_file + ".tar") + f = the_fs.open(folder) + else: + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + the_fs = fs.open_fs(dir_name) + f = the_fs.open(the_file_name) try: yield f finally: @@ -46,7 +55,14 @@ def open_file(path): @contextmanager def open_fs(path): path = to_unicode(path) - the_fs = fs.open_fs(path) + if "zip://" in path: + zip_file, folder = path.split(".zip") + the_fs = fs.open_fs(zip_file + ".zip") + elif "tar://" in path: + tar_file, folder = path.split(".tar") + the_fs = fs.open_fs(tar_file + ".tar") + else: + the_fs = fs.open_fs(path) try: yield the_fs finally: @@ -69,13 +85,14 @@ def read_unicode(path): else: dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) + with fs.open_fs(dir_name) as fs_system: with fs_system.open(the_file_name) as file_handle: return file_handle.read() @log_fs_failure -def read_binary(path): +def read_bytes(path): path = to_unicode(path) if "zip://" in path: zip_file, folder = path.split(".zip/") @@ -92,6 +109,9 @@ def read_binary(path): return fs_system.readbytes(the_file_name) +read_binary = read_bytes + + @log_fs_failure def write_bytes(filename, bytes_content): if "zip://" in filename: diff --git a/tests/fixtures/file_system/template-sources.tar b/tests/fixtures/file_system/template-sources.tar new file mode 100644 index 0000000000000000000000000000000000000000..2f13227fd560f0e893e00691f01d6332def4f280 GIT binary patch literal 10752 zcmeHNOOM(x5a!%p;TzHlg5x}pdac?^5A8pQ3>kuVLyT;PUH*N?z!I^n5*|sq-8u)H z#Ls*)-^_T7HyLz(QA(##LnC0_AvRkqYU&T)VLuWtQq%H7Re6(CsPRIov4VN2#RkT6x!#H*Yfmu{k1ORL z2R?K1??X|%$wK5X7RA`WYMpZfV=YTH2aQURr=UFZV(wfB%0CYGe1D(+K0D<jv8S*e3onKR)HZ zKSc}R_}>ot^`i(wgeJd!5+liyZ-8*3mr|ikkZLs|v(gZ=gq?-3O=dubs2Bp$A-3ar zzVBq;ZHEM23=~?(VnnngT-zQ-i7W#}zR3bB%@3)bxk|qmnJ8ei8_agxJ$lXt)eEbG z+A98QJ?uNK_WJlQ3}YAnAMxw*W*`40N9HEaBF~cpXIZ?d|6R)c zb!;2@Rd$@4#uV4ti!xBuL#L<_sV3aCc@vp4+r(TdZy&03gZwcF@yM`Ts}2{Ltn9 o`Vjvh#q-~4yZ05(ZS3KH^=9($;M_qkl5mqa1RMemfqoJA14BUqTL1t6 literal 0 HcmV?d00001 diff --git a/tests/fixtures/file_system/template-sources.zip b/tests/fixtures/file_system/template-sources.zip new file mode 100644 index 0000000000000000000000000000000000000000..5085110c10f6301afc5ac8775a078adb20101dfc GIT binary patch literal 2574 zcmbVMTSydP7@c)@rBq_0BrTD_BrJAjF}i3LD%s7&ZOsB-WSX7*cb%M_*=Asm`=SFFF7 zusP;z1mRcXjGsKf#cG>AA=lH&}oFsX8uz2P2LB<1?HogJ)%w`y1&{{t0392-**qo}N$x&RF& zia0(5jaiA|Jw(xihsWKmZS=V3-sq8<*B+|Xx9`GC4}a3OXbg$V`xAK1cx?o#SZ#_I zDHTPc0;?iQkzz836d1PWSX66Y!3k0;w!UvGNjj~04n+uY6#^;3QG6K7ifOuahTw2KZ8 z+s5V|inWrpiO!5_isan3c!fq%$=) z985NI)Ee4&0iTy`BgQO6p4fXJhUK*k zQ8OS-0b_9Lftm{lix@RS&|1li72jQk(K_s!#+hs~pPW|Jm~ndEz9p*a5-)$!ZUgBD zOO(&Dp&{*ZLo{ZczM*xFY?;guEt4-dLdJB{6M6)Mtg~)86lWoJLNT1rp-!C>zx@L( CwxeDE literal 0 HcmV?d00001 diff --git a/tests/test_file_system.py b/tests/test_file_system.py new file mode 100644 index 00000000..3550d1fe --- /dev/null +++ b/tests/test_file_system.py @@ -0,0 +1,106 @@ +import os +from moban import file_system +from nose.tools import eq_ + +LOCAL_FOLDER = 'tests/fixtures' +LOCAL_FILE = LOCAL_FOLDER + '/a.jj2' +ZIP_FILE = 'zip://tests/fixtures/file_system/template-sources.zip' +TAR_FILE = 'tar://tests/fixtures/file_system/template-sources.tar' + +FILE = 'file-in-template-sources-folder.txt' + +ZIP_URL = ZIP_FILE + '/' + FILE +TAR_URL = TAR_FILE + '/' + FILE + +TEST_SPECS = [ + LOCAL_FILE, + ZIP_URL, + TAR_URL +] + + +TEST_FS_SPECS = [ + LOCAL_FOLDER, + ZIP_FILE, + TAR_FILE +] + + +def test_open_file(): + for url in TEST_SPECS: + with file_system.open_file(url): + pass + + +def test_open_fs(): + for url in TEST_FS_SPECS: + with file_system.open_fs(url): + pass + + +TEST_FILE_CONTENT_SPECS = [ + [LOCAL_FILE, '{{key}} {{pass}}'], + [ZIP_URL, 'test file\n'], + [TAR_URL, 'test file\n'] +] + + +def test_read_unicode(): + for url, expected in TEST_FILE_CONTENT_SPECS: + content = file_system.read_unicode(url) + eq_(content, expected) + + +TEST_FILE_CONTENT_SPECS_BINARY = [ + [LOCAL_FILE, b'{{key}} {{pass}}'], + [ZIP_URL, b'test file\n'], + [TAR_URL, b'test file\n'] +] + + +def test_read_binary(): + for url, expected in TEST_FILE_CONTENT_SPECS_BINARY: + content = file_system.read_binary(url) + eq_(content, expected) + + +TEST_WRITE_BYTES_SPEC = [ + ['/tmp/test.binary', b'abc'], + ['zip:///tmp/test.zip/test.binary', b'abc'], + ['tar:///tmp/test.tar/test.binary', b'abc'] +] + + +def test_write_bytes(): + for url, content in TEST_WRITE_BYTES_SPEC: + file_system.write_bytes(url, content) + + for url, expected in TEST_WRITE_BYTES_SPEC: + content = file_system.read_bytes(url) + eq_(content, expected) + + for file_name in ['/tmp/test.binary', '/tmp/test.zip', '/tmp/test.tar']: + os.unlink(file_name) + + +TEST_DIR_SPEC = [ + [LOCAL_FOLDER, True], + [ZIP_FILE + '/dir-for-copying', True], + [TAR_FILE + '/dir-for-copying', True], + [ZIP_URL, False], + [TAR_URL, False], + [LOCAL_FILE, False] +] + + +def test_is_dir(): + for url, expected in TEST_DIR_SPEC: + status = file_system.is_dir(url) + eq_(status, expected) + + +def test_is_file(): + for url, is_dir in TEST_DIR_SPEC: + status = file_system.is_file(url) + expected = not is_dir + eq_(status, expected) From 6862d0e0edbff6505a22df6b2c3a32db1688d139 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 31 Jul 2019 08:40:23 +0100 Subject: [PATCH 089/167] :microscope: add file existence test --- tests/test_file_system.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_file_system.py b/tests/test_file_system.py index 3550d1fe..040d9928 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -104,3 +104,22 @@ def test_is_file(): status = file_system.is_file(url) expected = not is_dir eq_(status, expected) + + +TEST_URL_EXITENCE_SPEC = [ + [LOCAL_FOLDER, True], + [ZIP_FILE + '/dir-for-copying', True], + [TAR_FILE + '/dir-for-copying', True], + [ZIP_URL, True], + [TAR_URL, True], + [LOCAL_FILE, True], + #['zip://abc.zip', False], + #['tar://abc.tar', False], bug with fs.zipfs. raise it later + ['abcx', False] +] + + +def test_exists(): + for url, expected in TEST_URL_EXITENCE_SPEC: + status = file_system.exists(url) + eq_(status, expected) From 579da6754bfd6fa81270d7af64ea62f3f26b2c6d Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 31 Jul 2019 08:41:43 +0100 Subject: [PATCH 090/167] :shirt: coding style update --- tests/test_file_system.py | 63 +++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/tests/test_file_system.py b/tests/test_file_system.py index 040d9928..52bd341e 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -1,29 +1,22 @@ import os + from moban import file_system from nose.tools import eq_ -LOCAL_FOLDER = 'tests/fixtures' -LOCAL_FILE = LOCAL_FOLDER + '/a.jj2' -ZIP_FILE = 'zip://tests/fixtures/file_system/template-sources.zip' -TAR_FILE = 'tar://tests/fixtures/file_system/template-sources.tar' +LOCAL_FOLDER = "tests/fixtures" +LOCAL_FILE = LOCAL_FOLDER + "/a.jj2" +ZIP_FILE = "zip://tests/fixtures/file_system/template-sources.zip" +TAR_FILE = "tar://tests/fixtures/file_system/template-sources.tar" -FILE = 'file-in-template-sources-folder.txt' +FILE = "file-in-template-sources-folder.txt" -ZIP_URL = ZIP_FILE + '/' + FILE -TAR_URL = TAR_FILE + '/' + FILE +ZIP_URL = ZIP_FILE + "/" + FILE +TAR_URL = TAR_FILE + "/" + FILE -TEST_SPECS = [ - LOCAL_FILE, - ZIP_URL, - TAR_URL -] +TEST_SPECS = [LOCAL_FILE, ZIP_URL, TAR_URL] -TEST_FS_SPECS = [ - LOCAL_FOLDER, - ZIP_FILE, - TAR_FILE -] +TEST_FS_SPECS = [LOCAL_FOLDER, ZIP_FILE, TAR_FILE] def test_open_file(): @@ -39,9 +32,9 @@ def test_open_fs(): TEST_FILE_CONTENT_SPECS = [ - [LOCAL_FILE, '{{key}} {{pass}}'], - [ZIP_URL, 'test file\n'], - [TAR_URL, 'test file\n'] + [LOCAL_FILE, "{{key}} {{pass}}"], + [ZIP_URL, "test file\n"], + [TAR_URL, "test file\n"], ] @@ -52,9 +45,9 @@ def test_read_unicode(): TEST_FILE_CONTENT_SPECS_BINARY = [ - [LOCAL_FILE, b'{{key}} {{pass}}'], - [ZIP_URL, b'test file\n'], - [TAR_URL, b'test file\n'] + [LOCAL_FILE, b"{{key}} {{pass}}"], + [ZIP_URL, b"test file\n"], + [TAR_URL, b"test file\n"], ] @@ -65,9 +58,9 @@ def test_read_binary(): TEST_WRITE_BYTES_SPEC = [ - ['/tmp/test.binary', b'abc'], - ['zip:///tmp/test.zip/test.binary', b'abc'], - ['tar:///tmp/test.tar/test.binary', b'abc'] + ["/tmp/test.binary", b"abc"], + ["zip:///tmp/test.zip/test.binary", b"abc"], + ["tar:///tmp/test.tar/test.binary", b"abc"], ] @@ -79,17 +72,17 @@ def test_write_bytes(): content = file_system.read_bytes(url) eq_(content, expected) - for file_name in ['/tmp/test.binary', '/tmp/test.zip', '/tmp/test.tar']: + for file_name in ["/tmp/test.binary", "/tmp/test.zip", "/tmp/test.tar"]: os.unlink(file_name) TEST_DIR_SPEC = [ [LOCAL_FOLDER, True], - [ZIP_FILE + '/dir-for-copying', True], - [TAR_FILE + '/dir-for-copying', True], + [ZIP_FILE + "/dir-for-copying", True], + [TAR_FILE + "/dir-for-copying", True], [ZIP_URL, False], [TAR_URL, False], - [LOCAL_FILE, False] + [LOCAL_FILE, False], ] @@ -108,14 +101,14 @@ def test_is_file(): TEST_URL_EXITENCE_SPEC = [ [LOCAL_FOLDER, True], - [ZIP_FILE + '/dir-for-copying', True], - [TAR_FILE + '/dir-for-copying', True], + [ZIP_FILE + "/dir-for-copying", True], + [TAR_FILE + "/dir-for-copying", True], [ZIP_URL, True], [TAR_URL, True], [LOCAL_FILE, True], - #['zip://abc.zip', False], - #['tar://abc.tar', False], bug with fs.zipfs. raise it later - ['abcx', False] + # ['zip://abc.zip', False], + # ['tar://abc.tar', False], bug with fs.zipfs. raise it later + ["abcx", False], ] From 1d928baea90b0ddb8a649c308da5217e7698e6aa Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 31 Jul 2019 19:41:22 +0100 Subject: [PATCH 091/167] :bug: fs 2.2.0 is too low version to get current moban working --- .moban.cd/moban.yml | 2 +- min_requirements.txt | 2 +- requirements.txt | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 9db6158e..732605ab 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -29,7 +29,7 @@ dependencies: - crayons>= 0.1.0 - GitPython>=2.0.0 - git-url-parse>=1.2.2 - - fs>=2.2.0 + - fs>=2.4.6 description: Yet another jinja2 cli command for static text generation scm_host: github.com lint_command: make install_test format git-diff-check lint diff --git a/min_requirements.txt b/min_requirements.txt index 48290b81..a4db153d 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -8,4 +8,4 @@ appdirs==1.4.3 crayons== 0.1.0 GitPython==2.0.0 git-url-parse==1.2.2 -fs==2.2.0 +fs==2.4.6 diff --git a/requirements.txt b/requirements.txt index e368ffe7..17a27c58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ appdirs>=1.4.3 crayons>= 0.1.0 GitPython>=2.0.0 git-url-parse>=1.2.2 -fs>=2.2.0 +fs>=2.4.6 diff --git a/setup.py b/setup.py index 63982d39..228cfed9 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ "crayons>= 0.1.0", "GitPython>=2.0.0", "git-url-parse>=1.2.2", - "fs>=2.2.0", + "fs>=2.4.6", ] SETUP_COMMANDS = {} From fe42c43c0690981ce08a7c013b587084e3738863 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 31 Jul 2019 19:56:07 +0100 Subject: [PATCH 092/167] :shirt: pyfs2 on python 2 caught non-unicode again. #303 --- moban/file_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/file_system.py b/moban/file_system.py index 3c5c43c5..da654a9e 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -114,6 +114,7 @@ def read_bytes(path): @log_fs_failure def write_bytes(filename, bytes_content): + filename = to_unicode(filename) if "zip://" in filename: zip_file, folder = filename.split(".zip/") with fs.open_fs(zip_file + ".zip", create=True) as the_fs: @@ -124,7 +125,6 @@ def write_bytes(filename, bytes_content): with fs.open_fs(tar_file + ".tar", create=True) as the_fs: the_fs.writebytes(folder, bytes_content) else: - filename = to_unicode(filename) dir_name = fs.path.dirname(filename) the_file_name = fs.path.basename(filename) with fs.open_fs(dir_name) as the_fs: From 754e1a2736b0e7dedbc54cf6cf58c5dd6bdfef4d Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 1 Aug 2019 00:49:38 +0100 Subject: [PATCH 093/167] :microscope: full test suite for file system py --- tests/test_file_system.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/test_file_system.py b/tests/test_file_system.py index 52bd341e..36487a4b 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -116,3 +116,41 @@ def test_exists(): for url, expected in TEST_URL_EXITENCE_SPEC: status = file_system.exists(url) eq_(status, expected) + + +TEST_LIST_DIR_SPEC = [ + [LOCAL_FOLDER+'/file_system', ['template-sources.zip', 'template-sources.tar']], + [ZIP_FILE + "/dir-for-copying", ['afile.txt', 'sub_directory_is_not_copied']], + [TAR_FILE + "/dir-for-copying", ['afile.txt', 'sub_directory_is_not_copied']], +] + + +def test_list_dir(): + for url, expected in TEST_LIST_DIR_SPEC: + file_list = list(file_system.list_dir(url)) + eq_(file_list, expected) + + +TEST_FILE_PATH = [ + [LOCAL_FOLDER+'/file_system', + os.path.normpath(os.path.join(os.getcwd(), 'tests/fixtures/file_system'))] +] + + +def test_abspath(): + for path, expected in TEST_FILE_PATH: + url = file_system.abspath(path) + eq_(url, expected) + + +TEST_FILE_URL = [ + [LOCAL_FOLDER+'/file_system', + 'file://' + os.path.normpath(os.path.join(os.getcwd(), + 'tests/fixtures/file_system'))] +] + + +def test_fs_url(): + for path, expected in TEST_FILE_URL: + url = file_system.fs_url(path) + eq_(url, expected) From 7bc61650afdb93bbffcece1010ddf10bef8cb53d Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 1 Aug 2019 00:55:42 +0100 Subject: [PATCH 094/167] :green_heart: make unit test pass --- tests/test_file_system.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/tests/test_file_system.py b/tests/test_file_system.py index 36487a4b..eec43730 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -119,21 +119,34 @@ def test_exists(): TEST_LIST_DIR_SPEC = [ - [LOCAL_FOLDER+'/file_system', ['template-sources.zip', 'template-sources.tar']], - [ZIP_FILE + "/dir-for-copying", ['afile.txt', 'sub_directory_is_not_copied']], - [TAR_FILE + "/dir-for-copying", ['afile.txt', 'sub_directory_is_not_copied']], + [ + LOCAL_FOLDER + "/file_system", + ["template-sources.zip", "template-sources.tar"], + ], + [ + ZIP_FILE + "/dir-for-copying", + ["afile.txt", "sub_directory_is_not_copied"], + ], + [ + TAR_FILE + "/dir-for-copying", + ["afile.txt", "sub_directory_is_not_copied"], + ], ] def test_list_dir(): for url, expected in TEST_LIST_DIR_SPEC: - file_list = list(file_system.list_dir(url)) - eq_(file_list, expected) + file_list = sorted(list(file_system.list_dir(url))) + eq_(file_list, sorted(expected)) TEST_FILE_PATH = [ - [LOCAL_FOLDER+'/file_system', - os.path.normpath(os.path.join(os.getcwd(), 'tests/fixtures/file_system'))] + [ + LOCAL_FOLDER + "/file_system", + os.path.normpath( + os.path.join(os.getcwd(), "tests/fixtures/file_system") + ), + ] ] @@ -144,9 +157,13 @@ def test_abspath(): TEST_FILE_URL = [ - [LOCAL_FOLDER+'/file_system', - 'file://' + os.path.normpath(os.path.join(os.getcwd(), - 'tests/fixtures/file_system'))] + [ + LOCAL_FOLDER + "/file_system", + "file://" + + os.path.normpath( + os.path.join(os.getcwd(), "tests/fixtures/file_system") + ), + ] ] From b5db6054cacb30103cfa8d5d4a9b2f8d44112a7b Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 1 Aug 2019 01:01:37 +0100 Subject: [PATCH 095/167] :green_heart: make windows build pass --- tests/test_file_system.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_file_system.py b/tests/test_file_system.py index eec43730..46a62da5 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -58,9 +58,9 @@ def test_read_binary(): TEST_WRITE_BYTES_SPEC = [ - ["/tmp/test.binary", b"abc"], - ["zip:///tmp/test.zip/test.binary", b"abc"], - ["tar:///tmp/test.tar/test.binary", b"abc"], + ["test.binary", b"abc"], + ["zip://test.zip/test.binary", b"abc"], + ["tar://test.tar/test.binary", b"abc"], ] @@ -72,7 +72,7 @@ def test_write_bytes(): content = file_system.read_bytes(url) eq_(content, expected) - for file_name in ["/tmp/test.binary", "/tmp/test.zip", "/tmp/test.tar"]: + for file_name in ["test.binary", "test.zip", "test.tar"]: os.unlink(file_name) @@ -170,4 +170,4 @@ def test_abspath(): def test_fs_url(): for path, expected in TEST_FILE_URL: url = file_system.fs_url(path) - eq_(url, expected) + eq_(url, expected.replace('\\', '/')) From 654692b1718209fe6ba2af5c1c225778d273ed0f Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 1 Aug 2019 20:35:23 +0100 Subject: [PATCH 096/167] :hammer: code refactor file system py --- moban/file_system.py | 124 +++++++++++++++----------------------- tests/test_file_system.py | 2 +- 2 files changed, 50 insertions(+), 76 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index da654a9e..89f1e2c4 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -1,6 +1,7 @@ import sys import logging from contextlib import contextmanager +from urllib.parse import urlparse import fs import fs.path @@ -27,17 +28,35 @@ def wrapper(*args, **kwds): return wrapper +def is_zip_alike_url(url): + specs = ["zip://", "tar://"] + for prefix in specs: + if url.startswith(prefix): + return True + else: + return False + + +def url_split(url): + result = urlparse(url) + + if url.endswith(result.scheme): + url_to_file = url + path = None + else: + url_to_file, path = url.split(result.scheme + "/") + url_to_file = url_to_file + result.scheme + + return url_to_file, path + + @log_fs_failure @contextmanager def open_file(path): path = to_unicode(path) - if "zip://" in path: - zip_file, folder = path.split(".zip/") - the_fs = fs.open_fs(zip_file + ".zip") - f = the_fs.open(folder) - elif "tar://" in path: - tar_file, folder = path.split(".tar/") - the_fs = fs.open_fs(tar_file + ".tar") + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + the_fs = fs.open_fs(zip_file) f = the_fs.open(folder) else: dir_name = fs.path.dirname(path) @@ -55,12 +74,9 @@ def open_file(path): @contextmanager def open_fs(path): path = to_unicode(path) - if "zip://" in path: - zip_file, folder = path.split(".zip") - the_fs = fs.open_fs(zip_file + ".zip") - elif "tar://" in path: - tar_file, folder = path.split(".tar") - the_fs = fs.open_fs(tar_file + ".tar") + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + the_fs = fs.open_fs(zip_file) else: the_fs = fs.open_fs(path) try: @@ -72,14 +88,9 @@ def open_fs(path): @log_fs_failure def read_unicode(path): path = to_unicode(path) - if "zip://" in path: - zip_file, folder = path.split(".zip/") - with fs.open_fs(zip_file + ".zip") as the_fs: - with the_fs.open(folder) as file_handle: - return file_handle.read() - elif "tar://" in path: - tar_file, folder = path.split(".tar/") - with fs.open_fs(tar_file + ".tar") as the_fs: + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + with fs.open_fs(zip_file) as the_fs: with the_fs.open(folder) as file_handle: return file_handle.read() else: @@ -94,13 +105,9 @@ def read_unicode(path): @log_fs_failure def read_bytes(path): path = to_unicode(path) - if "zip://" in path: - zip_file, folder = path.split(".zip/") - with fs.open_fs(zip_file + ".zip") as the_fs: - return the_fs.readbytes(folder) - elif "tar://" in path: - tar_file, folder = path.split(".tar/") - with fs.open_fs(tar_file + ".tar") as the_fs: + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + with fs.open_fs(zip_file) as the_fs: return the_fs.readbytes(folder) else: dir_name = fs.path.dirname(path) @@ -115,14 +122,9 @@ def read_bytes(path): @log_fs_failure def write_bytes(filename, bytes_content): filename = to_unicode(filename) - if "zip://" in filename: - zip_file, folder = filename.split(".zip/") - with fs.open_fs(zip_file + ".zip", create=True) as the_fs: - the_fs.writebytes(folder, bytes_content) - - elif "tar://" in filename: - tar_file, folder = filename.split(".tar/") - with fs.open_fs(tar_file + ".tar", create=True) as the_fs: + if "://" in filename: + zip_file, folder = url_split(filename) + with fs.open_fs(zip_file, create=True) as the_fs: the_fs.writebytes(folder, bytes_content) else: dir_name = fs.path.dirname(filename) @@ -133,14 +135,9 @@ def write_bytes(filename, bytes_content): @log_fs_failure def is_dir(path): - if "zip://" in path: - zip_file, folder = path.split(".zip/") - with fs.open_fs(zip_file + ".zip") as the_fs: - return the_fs.isdir(to_unicode(folder)) - - if "tar://" in path: - zip_file, folder = path.split(".tar/") - with fs.open_fs(zip_file + ".tar") as the_fs: + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + with fs.open_fs(zip_file) as the_fs: return the_fs.isdir(to_unicode(folder)) path = to_unicode(path) @@ -163,38 +160,15 @@ def is_file(path): def exists(path): path = to_unicode(path) - if "zip://" in path: - if path.endswith(".zip"): - zip_file, folder = path, "/" - try: - with fs.open_fs(zip_file) as the_fs: - return True - except fs.errors.CreateFailed: - return False - else: - zip_file, folder = path.split(".zip/") - try: - with fs.open_fs(zip_file + ".zip") as the_fs: - return the_fs.exists(folder) - except fs.errors.CreateFailed: - return False - - if "tar://" in path: - if path.endswith(".tar"): - zip_file, folder = path, "/" - try: - with fs.open_fs(zip_file) as the_fs: - return True - except fs.errors.CreateFailed: - return False - else: - zip_file, folder = path.split(".tar/") - try: - with fs.open_fs(zip_file + ".tar") as the_fs: + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + try: + with fs.open_fs(zip_file) as the_fs: + if folder: return the_fs.exists(folder) - except fs.errors.CreateFailed: - return False - + return True + except fs.errors.CreateFailed: + return False dir_name = fs.path.dirname(path) the_file_name = fs.path.basename(path) diff --git a/tests/test_file_system.py b/tests/test_file_system.py index 46a62da5..3e977bba 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -170,4 +170,4 @@ def test_abspath(): def test_fs_url(): for path, expected in TEST_FILE_URL: url = file_system.fs_url(path) - eq_(url, expected.replace('\\', '/')) + eq_(url, expected.replace("\\", "/")) From b26ceef2c50de154fca5ad2553c705b3e8d95d15 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 1 Aug 2019 20:48:13 +0100 Subject: [PATCH 097/167] :hammer: further refactoring to reduce the length of file system --- moban/file_system.py | 74 +++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index 89f1e2c4..44792c36 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -50,6 +50,29 @@ def url_split(url): return url_to_file, path +def _path_split(url_or_path): + url_or_path = to_unicode(url_or_path) + if is_zip_alike_url(url_or_path): + return url_split(url_or_path) + else: + return fs.path.dirname(url_or_path), fs.path.basename(url_or_path) + + +@log_fs_failure +@contextmanager +def open_fs(path): + path = to_unicode(path) + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + the_fs = fs.open_fs(zip_file) + else: + the_fs = fs.open_fs(path) + try: + yield the_fs + finally: + the_fs.close() + + @log_fs_failure @contextmanager def open_file(path): @@ -72,48 +95,34 @@ def open_file(path): @log_fs_failure @contextmanager -def open_fs(path): +def open_binary_file(path): path = to_unicode(path) if is_zip_alike_url(path): zip_file, folder = url_split(path) the_fs = fs.open_fs(zip_file) + f = the_fs.openbin(folder) else: - the_fs = fs.open_fs(path) + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + the_fs = fs.open_fs(dir_name) + f = the_fs.openbin(the_file_name) try: - yield the_fs + yield f finally: + f.close() the_fs.close() @log_fs_failure def read_unicode(path): - path = to_unicode(path) - if is_zip_alike_url(path): - zip_file, folder = url_split(path) - with fs.open_fs(zip_file) as the_fs: - with the_fs.open(folder) as file_handle: - return file_handle.read() - else: - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - - with fs.open_fs(dir_name) as fs_system: - with fs_system.open(the_file_name) as file_handle: - return file_handle.read() + with open_file(path) as file_handle: + return file_handle.read() @log_fs_failure def read_bytes(path): - path = to_unicode(path) - if is_zip_alike_url(path): - zip_file, folder = url_split(path) - with fs.open_fs(zip_file) as the_fs: - return the_fs.readbytes(folder) - else: - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - with fs.open_fs(dir_name) as fs_system: - return fs_system.readbytes(the_file_name) + with open_binary_file(path) as file_handle: + return file_handle.read() read_binary = read_bytes @@ -135,16 +144,9 @@ def write_bytes(filename, bytes_content): @log_fs_failure def is_dir(path): - if is_zip_alike_url(path): - zip_file, folder = url_split(path) - with fs.open_fs(zip_file) as the_fs: - return the_fs.isdir(to_unicode(folder)) - - path = to_unicode(path) - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - with fs.open_fs(dir_name) as the_fs: - return the_fs.isdir(the_file_name) + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + return the_fs.isdir(path) @log_fs_failure From 6ff7499fa2e6d769e5131d11a42a64852d53ccf2 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 1 Aug 2019 20:51:42 +0100 Subject: [PATCH 098/167] :bug: fix python 2 compactibility. urlparse #303 --- moban/file_system.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/moban/file_system.py b/moban/file_system.py index 44792c36..479b5680 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -1,7 +1,10 @@ import sys import logging from contextlib import contextmanager -from urllib.parse import urlparse +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse import fs import fs.path From e1f4efa9645f37c2588628f115ee4ec533cd2ea7 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 1 Aug 2019 21:00:58 +0100 Subject: [PATCH 099/167] :tractor: further shrink the file system --- moban/file_system.py | 97 +++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 51 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index 479b5680..e7e6f2c9 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -1,13 +1,15 @@ import sys import logging from contextlib import contextmanager + +import fs +import fs.path + try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse -import fs -import fs.path PY2 = sys.version_info[0] == 2 LOG = logging.getLogger(__name__) @@ -31,36 +33,6 @@ def wrapper(*args, **kwds): return wrapper -def is_zip_alike_url(url): - specs = ["zip://", "tar://"] - for prefix in specs: - if url.startswith(prefix): - return True - else: - return False - - -def url_split(url): - result = urlparse(url) - - if url.endswith(result.scheme): - url_to_file = url - path = None - else: - url_to_file, path = url.split(result.scheme + "/") - url_to_file = url_to_file + result.scheme - - return url_to_file, path - - -def _path_split(url_or_path): - url_or_path = to_unicode(url_or_path) - if is_zip_alike_url(url_or_path): - return url_split(url_or_path) - else: - return fs.path.dirname(url_or_path), fs.path.basename(url_or_path) - - @log_fs_failure @contextmanager def open_fs(path): @@ -154,11 +126,9 @@ def is_dir(path): @log_fs_failure def is_file(path): - path = to_unicode(path) - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - with fs.open_fs(dir_name) as the_fs: - return the_fs.isfile(the_file_name) + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + return the_fs.isfile(path) @log_fs_failure @@ -187,34 +157,59 @@ def exists(path): @log_fs_failure def list_dir(path): path = to_unicode(path) - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - with fs.open_fs(dir_name) as fs_system: - for file_name in fs_system.listdir(the_file_name): + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + for file_name in the_fs.listdir(path): yield file_name @log_fs_failure def abspath(path): path = to_unicode(path) - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - - with fs.open_fs(dir_name) as the_fs: - return the_fs.getsyspath(the_file_name) + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + return the_fs.getsyspath(path) @log_fs_failure def fs_url(path): path = to_unicode(path) - dir_name = fs.path.dirname(path) - the_file_name = fs.path.basename(path) - - with fs.open_fs(dir_name) as the_fs: - return the_fs.geturl(the_file_name) + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + return the_fs.geturl(path) def to_unicode(path): if PY2 and path.__class__.__name__ != "unicode": return u"".__class__(path) return path + + +def is_zip_alike_url(url): + specs = ["zip://", "tar://"] + for prefix in specs: + if url.startswith(prefix): + return True + else: + return False + + +def url_split(url): + result = urlparse(url) + + if url.endswith(result.scheme): + url_to_file = url + path = None + else: + url_to_file, path = url.split(result.scheme + "/") + url_to_file = url_to_file + result.scheme + + return url_to_file, path + + +def _path_split(url_or_path): + url_or_path = to_unicode(url_or_path) + if is_zip_alike_url(url_or_path): + return url_split(url_or_path) + else: + return fs.path.dirname(url_or_path), fs.path.basename(url_or_path) From adef81319756041f4885dcbaf182fc5ba1979d9f Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 4 Aug 2019 20:17:22 +0100 Subject: [PATCH 100/167] :sparkles: pyfs2 review feedback for better zip/tar fs url. https://github.com/PyFilesystem/pyfilesystem2/pull/330 --- .../.moban.yml | 4 ++-- .../.moban.yml | 12 +++++------ moban/adapter/tarfs.py | 2 +- moban/adapter/zipfs.py | 2 +- moban/buffered_writer.py | 3 +-- moban/file_system.py | 3 +-- moban/mobanfile/templates.py | 8 ++++---- moban/utils.py | 7 +++++-- rnd_requirements.txt | 3 --- tests/test_docs.py | 15 ++++++++------ tests/test_file_system.py | 20 +++++++++---------- 11 files changed, 40 insertions(+), 39 deletions(-) diff --git a/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml b/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml index 8e124971..1caa249e 100644 --- a/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml +++ b/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml @@ -5,9 +5,9 @@ configuration: - cool-templates - '.' targets: - - output: 'tar://a.tar/a.output' + - output: 'tar://a.tar!/a.output' configuration: data.yml template: template.in.zip.jj2 - - output: 'zip://a.zip/a.output2' + - output: 'zip://a.zip!/a.output2' configuration: data2.yml template: subfolder/template.in.zip.jj2 diff --git a/docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml b/docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml index aefbad15..4ae79020 100644 --- a/docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml +++ b/docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml @@ -2,18 +2,18 @@ configuration: template_dir: - "zip://template-sources.zip" targets: - - output: "zip://my.zip/simple.file.copy" + - output: "zip://my.zip!/simple.file.copy" template: file-in-template-sources-folder.txt template_type: copy - - output: "zip://my.zip/target_without_template_type" + - output: "zip://my.zip!/target_without_template_type" template: file_extension_will_trigger.copy - - "zip://my.zip/target_in_short_form": as_long_as_this_one_has.copy - - output: "zip://my.zip/misc-1-copying/can-create-folder/if-not-exists.txt" + - "zip://my.zip!/target_in_short_form": as_long_as_this_one_has.copy + - output: "zip://my.zip!/misc-1-copying/can-create-folder/if-not-exists.txt" template: file-in-template-sources-folder.txt template_type: copy - - output: "zip://my.zip/test-dir" + - output: "zip://my.zip!/test-dir" template: dir-for-copying template_type: copy - - output: "zip://my.zip/test-recursive-dir" + - output: "zip://my.zip!/test-recursive-dir" template: dir-for-recusive-copying/** template_type: copy diff --git a/moban/adapter/tarfs.py b/moban/adapter/tarfs.py index 87f391e4..0174b990 100644 --- a/moban/adapter/tarfs.py +++ b/moban/adapter/tarfs.py @@ -6,7 +6,7 @@ class EnhancedReadTarFS(ReadTarFS): def geturl(self, path, purpose="download"): - return "tar://%s/%s" % (self._file, path) + return "tar://%s!/%s" % (self._file, path) class EnhancedTarFS(TarFS): diff --git a/moban/adapter/zipfs.py b/moban/adapter/zipfs.py index f792f628..a9050275 100644 --- a/moban/adapter/zipfs.py +++ b/moban/adapter/zipfs.py @@ -5,7 +5,7 @@ class EnhancedReadZipFS(ReadZipFS): def geturl(self, path, purpose="download"): - return "zip://%s/%s" % (self._file, path) + return "zip://%s!/%s" % (self._file, path) class EnhancedZipFS(ZipFS): diff --git a/moban/buffered_writer.py b/moban/buffered_writer.py index e000dce9..49cbdb3e 100644 --- a/moban/buffered_writer.py +++ b/moban/buffered_writer.py @@ -15,8 +15,7 @@ def write_file_out(self, filename, content): utils.write_file_out(filename, content) def write_file_out_to_zip(self, filename, content): - zip_file, file_name = filename.split(".zip/") - zip_file = zip_file + ".zip" + zip_file, file_name = file_system.url_split(filename) if zip_file not in self.fs_list: self.fs_list[zip_file] = fs.open_fs( file_system.to_unicode(zip_file), create=True diff --git a/moban/file_system.py b/moban/file_system.py index e7e6f2c9..da0bcfa0 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -201,8 +201,7 @@ def url_split(url): url_to_file = url path = None else: - url_to_file, path = url.split(result.scheme + "/") - url_to_file = url_to_file + result.scheme + url_to_file, path = url.split("!/") return url_to_file, path diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 079f13e9..29e1376b 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -68,8 +68,8 @@ def _list_dir_files(source, actual_source_path, dest): def _list_zip_dir_files(source, actual_source_path, dest): - zip_file, folder = actual_source_path.split(".zip/") - with file_system.open_fs(zip_file + ".zip") as fs_handle: + zip_file, folder = file_system.url_split(actual_source_path) + with file_system.open_fs(zip_file) as fs_handle: for file_name in fs_handle.listdir(file_system.to_unicode(folder)): if fs_handle.isfile( file_system.to_unicode(folder + "/" + file_name) @@ -102,10 +102,10 @@ def _listing_directory_files_recusively(source, actual_source_path, dest): def _listing_zip_directory_files_recusively(source, actual_source_path, dest): - zip_file, folder = actual_source_path.split(".zip/") + zip_file, folder = file_system.url_split(actual_source_path) print(actual_source_path) - with file_system.open_fs(zip_file + ".zip") as fs_handle: + with file_system.open_fs(zip_file) as fs_handle: for file_name in fs_handle.listdir(file_system.to_unicode(folder)): src_file_under_dir = source + "/" + file_name dest_file_under_dir = dest + "/" + file_name diff --git a/moban/utils.py b/moban/utils.py index 07641189..8f98c77a 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -148,7 +148,10 @@ def verify_the_existence_of_directories(dirs): def find_file_in_template_dirs(src, template_dirs): LOG.debug(template_dirs) for folder in template_dirs: - if file_system.exists(folder + "/" + src): - return folder + "/" + src + path = folder + "/" + src + if "zip://" in folder or "tar://" in folder: + path = folder + '!/' + src + if file_system.exists(path): + return path else: return None diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 52963a4d..3f889887 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,5 +1,2 @@ https://github.com/moremoban/jinja2-fsloader/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip - - - diff --git a/tests/test_docs.py b/tests/test_docs.py index 60ee3a65..0c0ab1c6 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,6 +1,7 @@ import os import sys from textwrap import dedent +from fs.opener.parse import parse_fs_url from mock import patch from moban import file_system @@ -117,7 +118,7 @@ def test_level_20(self): ) folder = "level-20-templates-configs-in-zip-or-tar" self._raw_moban_with_fs( - ["moban"], folder, expected, "zip://a.zip/a.output2" + ["moban"], folder, expected, "zip://a.zip!/a.output2" ) def test_level_7(self): @@ -257,13 +258,13 @@ def test_level_21_copy_templates_into_zips(self): folder = "level-21-copy-templates-into-an-alien-file-system" criterias = [ - ["zip://my.zip/simple.file", expected], + ["zip://my.zip!/simple.file", expected], [ - "zip://my.zip/target_without_template_type", + "zip://my.zip!/target_without_template_type", "file extension will trigger copy engine\n", ], [ - "zip://my.zip/target_in_short_form", + "zip://my.zip!/target_in_short_form", ( "it is OK to have a short form, " + "but the file to be 'copied' shall have 'copy' extension, " @@ -335,7 +336,8 @@ def _raw_moban_with_fs(self, args, folder, expected, output): with patch.object(sys, "argv", args): main() _verify_content_with_fs(output, expected) - os.unlink(output.split("/")[2]) # fixme later + result = parse_fs_url(output) + os.unlink(result.resource) # delete the zip file def _raw_moban_with_fs2(self, args, folder, criterias): os.chdir(os.path.join("docs", folder)) @@ -344,7 +346,8 @@ def _raw_moban_with_fs2(self, args, folder, criterias): for output, expected in criterias: _verify_content_with_fs(output, expected) - os.unlink(output.split("/")[2]) # fixme later + result = parse_fs_url(output) + os.unlink(result.resource) # delete the zip file def tearDown(self): if os.path.exists(".moban.hashes"): diff --git a/tests/test_file_system.py b/tests/test_file_system.py index 3e977bba..e41fc189 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -10,8 +10,8 @@ FILE = "file-in-template-sources-folder.txt" -ZIP_URL = ZIP_FILE + "/" + FILE -TAR_URL = TAR_FILE + "/" + FILE +ZIP_URL = ZIP_FILE + "!/" + FILE +TAR_URL = TAR_FILE + "!/" + FILE TEST_SPECS = [LOCAL_FILE, ZIP_URL, TAR_URL] @@ -59,8 +59,8 @@ def test_read_binary(): TEST_WRITE_BYTES_SPEC = [ ["test.binary", b"abc"], - ["zip://test.zip/test.binary", b"abc"], - ["tar://test.tar/test.binary", b"abc"], + ["zip://test.zip!/test.binary", b"abc"], + ["tar://test.tar!/test.binary", b"abc"], ] @@ -78,8 +78,8 @@ def test_write_bytes(): TEST_DIR_SPEC = [ [LOCAL_FOLDER, True], - [ZIP_FILE + "/dir-for-copying", True], - [TAR_FILE + "/dir-for-copying", True], + [ZIP_FILE + "!/dir-for-copying", True], + [TAR_FILE + "!/dir-for-copying", True], [ZIP_URL, False], [TAR_URL, False], [LOCAL_FILE, False], @@ -101,8 +101,8 @@ def test_is_file(): TEST_URL_EXITENCE_SPEC = [ [LOCAL_FOLDER, True], - [ZIP_FILE + "/dir-for-copying", True], - [TAR_FILE + "/dir-for-copying", True], + [ZIP_FILE + "!/dir-for-copying", True], + [TAR_FILE + "!/dir-for-copying", True], [ZIP_URL, True], [TAR_URL, True], [LOCAL_FILE, True], @@ -124,11 +124,11 @@ def test_exists(): ["template-sources.zip", "template-sources.tar"], ], [ - ZIP_FILE + "/dir-for-copying", + ZIP_FILE + "!/dir-for-copying", ["afile.txt", "sub_directory_is_not_copied"], ], [ - TAR_FILE + "/dir-for-copying", + TAR_FILE + "!/dir-for-copying", ["afile.txt", "sub_directory_is_not_copied"], ], ] From 40ee4614fbe414c12d63114388571dff6a735f61 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 4 Aug 2019 20:35:45 +0100 Subject: [PATCH 101/167] :shirt: update coding style --- moban/utils.py | 2 +- tests/test_docs.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 8f98c77a..4a6c1cc3 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -150,7 +150,7 @@ def find_file_in_template_dirs(src, template_dirs): for folder in template_dirs: path = folder + "/" + src if "zip://" in folder or "tar://" in folder: - path = folder + '!/' + src + path = folder + "!/" + src if file_system.exists(path): return path else: diff --git a/tests/test_docs.py b/tests/test_docs.py index 0c0ab1c6..003722bb 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,13 +1,14 @@ import os import sys from textwrap import dedent -from fs.opener.parse import parse_fs_url from mock import patch from moban import file_system from moban.main import main from nose.tools import eq_ +from fs.opener.parse import parse_fs_url + def custom_dedent(long_texts): refined = dedent(long_texts) From b7948aaae9bd9a8ee711f634771a96a8f26cb7e5 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 5 Aug 2019 00:30:40 +0100 Subject: [PATCH 102/167] :fire: use jinja2-fsloader as it is. https://github.com/althonos/jinja2-fsloader/pull/3 --- .moban.cd/moban.yml | 1 + min_requirements.txt | 1 + moban/jinja2/engine.py | 8 +++++++- requirements.txt | 1 + rnd_requirements.txt | 1 - setup.py | 1 + 6 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 732605ab..369bfc28 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -30,6 +30,7 @@ dependencies: - GitPython>=2.0.0 - git-url-parse>=1.2.2 - fs>=2.4.6 + - jinja2-fsloader>=0.1.3 description: Yet another jinja2 cli command for static text generation scm_host: github.com lint_command: make install_test format git-diff-check lint diff --git a/min_requirements.txt b/min_requirements.txt index a4db153d..6921d623 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -9,3 +9,4 @@ crayons== 0.1.0 GitPython==2.0.0 git-url-parse==1.2.2 fs==2.4.6 +jinja2-fsloader==0.1.3 diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index 87c01b57..e82acfc4 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -2,6 +2,8 @@ import logging from importlib import import_module +import fs +from fs.multifs import MultiFS from moban import constants, exceptions from jinja2 import Template, Environment from lml.loader import scan_plugins_regex @@ -68,7 +70,11 @@ def __init__(self, template_dirs, options=None): LOG.debug("Jinja template dirs: %s", template_dirs) load_jinja2_extensions() self.template_dirs = template_dirs - template_loader = FSLoader(template_dirs) + filesystem = MultiFS() + for template_dir in template_dirs: + filesystem.add_fs(template_dir, fs.open_fs(template_dir)) + + template_loader = FSLoader(filesystem) env_params = dict( loader=template_loader, keep_trailing_newline=True, diff --git a/requirements.txt b/requirements.txt index 17a27c58..7e261a09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ crayons>= 0.1.0 GitPython>=2.0.0 git-url-parse>=1.2.2 fs>=2.4.6 +jinja2-fsloader>=0.1.3 diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 3f889887..86b965ae 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,2 +1 @@ -https://github.com/moremoban/jinja2-fsloader/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip diff --git a/setup.py b/setup.py index 228cfed9..43451c54 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ "GitPython>=2.0.0", "git-url-parse>=1.2.2", "fs>=2.4.6", + "jinja2-fsloader>=0.1.3", ] SETUP_COMMANDS = {} From 1413e7d53ba72063f1d513ace36c73ed38900551 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 5 Aug 2019 08:27:14 +0100 Subject: [PATCH 103/167] :fire: remove internal pypifs opener with external and universal pypi opened. the legacy code is kept for backward compactibility --- .../.moban.yml | 4 ++-- moban/fs_openers.py | 13 ------------- rnd_requirements.txt | 2 ++ 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/docs/level-9-moban-dependency-as-pypi-package/.moban.yml b/docs/level-9-moban-dependency-as-pypi-package/.moban.yml index f9e7bc46..5618d05f 100644 --- a/docs/level-9-moban-dependency-as-pypi-package/.moban.yml +++ b/docs/level-9-moban-dependency-as-pypi-package/.moban.yml @@ -2,10 +2,10 @@ requires: - pypi-mobans-pkg configuration: template_dir: - - "pypi://pypi-mobans-pkg/templates" + - "pypi://pypi-mobans-pkg/resources/templates" - local configuration: config.yml - configuration_dir: "pypi://pypi-mobans-pkg/config" + configuration_dir: "pypi://pypi-mobans-pkg/resources/config" targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 diff --git a/moban/fs_openers.py b/moban/fs_openers.py index d4cc484b..d31ab296 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -2,7 +2,6 @@ from moban.adapter.osfs import EnhancedOSFS from moban.adapter.tarfs import EnhancedTarFS from moban.adapter.zipfs import EnhancedZipFS -from moban.plugins.library import LIBRARIES import fs import fs.path @@ -10,18 +9,6 @@ from fs.opener.registry import registry -@registry.install -class PypiFSOpener(Opener): - protocols = ["pypi"] - - def open_fs(self, fs_url, parse_result, writeable, create, cwd): - pypi_package_name, _, dir_path = parse_result.resource.partition("/") - library_path = LIBRARIES.resource_path_of(pypi_package_name) - root_path = fs.path.join(library_path, dir_path) - osfs = EnhancedOSFS(root_path=root_path) - return osfs - - @registry.install class RepoFSOpener(Opener): protocols = ["repo"] diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 86b965ae..9272e949 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1 +1,3 @@ https://github.com/moremoban/moban-handlebars/archive/dev.zip +https://github.com/moremoban/pypifs/archive/master.zip + From 46f584213a4bfd8413232d26abdfacf6d13a7ccd Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 6 Aug 2019 23:02:40 +0100 Subject: [PATCH 104/167] book: update documenation --- .moban.cd/moban.yml | 5 ++--- .../README.rst | 12 ++++++------ .../README.rst | 2 +- setup.py | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 369bfc28..16e7e55c 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -9,9 +9,8 @@ current_version: 0.6.0 release: 0.5.0 branch: master master: index -entry_points: - console_scripts: - - "moban = moban.main:main" +command_line_interface: "moban" +entry_point: "moban.main:main" company: Onni Software Ltd. copyright_year: 2017-2019 copyright: 2017-2019 Onni Software Ltd. and its contributors diff --git a/docs/level-21-copy-templates-into-an-alien-file-system/README.rst b/docs/level-21-copy-templates-into-an-alien-file-system/README.rst index 684367a8..83e20dc8 100644 --- a/docs/level-21-copy-templates-into-an-alien-file-system/README.rst +++ b/docs/level-21-copy-templates-into-an-alien-file-system/README.rst @@ -8,22 +8,22 @@ all templates are copied to. Explicit syntax:: targets: - - output: "zip://your.zip/explicit" - template: template_file - template_type: copy + - output: "zip://your.zip/explicit" + template: template_file + template_type: copy Implicit syntax:: targets: - - output: "zip://your.zip/implicit" - template: template_file.copy + - output: "zip://your.zip/implicit" + template: template_file.copy Shorthand syntax:: targets: - - "zip://your.zip/shorthand": template_file.copy + - "zip://your.zip/shorthand": template_file.copy No implicit nor short hand syntax for the following directory copying unless diff --git a/docs/level-9-moban-dependency-as-pypi-package/README.rst b/docs/level-9-moban-dependency-as-pypi-package/README.rst index cd096b67..ebf4e913 100644 --- a/docs/level-9-moban-dependency-as-pypi-package/README.rst +++ b/docs/level-9-moban-dependency-as-pypi-package/README.rst @@ -12,7 +12,7 @@ into a pypi package and distribute it to the world of moban. Here are the sample file:: requires: - - pypi-mobans-pkg + - pypi-mobans-pkg configuration: template_dir: - "pypi://pypi-mobans-pkg/templates" diff --git a/setup.py b/setup.py index 43451c54..46a52b34 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ LICENSE = "MIT" ENTRY_POINTS = { "console_scripts": [ - "moban = moban.main:main", + "moban = moban.main:main" ], } DESCRIPTION = ( From ec7f375710b1ecb60a84b2ba5301d23ae80d14b5 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 6 Aug 2019 23:13:33 +0100 Subject: [PATCH 105/167] :art: install rnd version of fs2 for debugging windows build failure --- rnd_requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 9272e949..efab09e1 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,3 +1,5 @@ +https://github.com/moremoban/pyfilesystem2/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip https://github.com/moremoban/pypifs/archive/master.zip + From 9af40049a40aedd24ab12bcd2b121496c4087c03 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 6 Aug 2019 23:25:45 +0100 Subject: [PATCH 106/167] :art: try use local adapted version of osfs --- moban/adapter/osfs.py | 19 ++++++++++++++++++- rnd_requirements.txt | 1 - 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/moban/adapter/osfs.py b/moban/adapter/osfs.py index 36c75434..ae637bb6 100644 --- a/moban/adapter/osfs.py +++ b/moban/adapter/osfs.py @@ -1,9 +1,26 @@ +import platform import fs from fs.osfs import OSFS +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + +_WINDOWS_PLATFORM = platform.system() == "Windows" class EnhancedOSFS(OSFS): def geturl(self, path, purpose="download"): if purpose != "download": raise fs.errors.NoURL(path, purpose) - return "file://" + self.getsyspath(path).replace("\\", "/") + + sys_path = self.getsyspath(path) + if _WINDOWS_PLATFORM and ':' in sys_path: + drive_letter, path = sys_path.split(':', 1) + path = path.replace("\\", "/") + path = quote(path) + url_path = "{}:{}".format(drive_letter, path) + else: + sys_path = sys_path.replace("\\", "/") + url_path = quote(sys_path) + return "file://" + url_path diff --git a/rnd_requirements.txt b/rnd_requirements.txt index efab09e1..d8fde3d4 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,4 +1,3 @@ -https://github.com/moremoban/pyfilesystem2/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip https://github.com/moremoban/pypifs/archive/master.zip From 65fac55c6933bcd204f2ba4f8e8fe2ae1f104fe3 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 6 Aug 2019 23:33:06 +0100 Subject: [PATCH 107/167] :ambulance: it looks like pypifs have to pin on the next version of pyfilesystem 2 --- rnd_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/rnd_requirements.txt b/rnd_requirements.txt index d8fde3d4..efab09e1 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,3 +1,4 @@ +https://github.com/moremoban/pyfilesystem2/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip https://github.com/moremoban/pypifs/archive/master.zip From 32f5ebaf461d95294b28a05dbb5591800eacccd4 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 7 Aug 2019 07:33:35 +0100 Subject: [PATCH 108/167] :fire: remove adapation code and use next version of pyfilesystem2. https://github.com/PyFilesystem/pyfilesystem2/pull/330 --- moban/adapter/osfs.py | 26 -------------------- moban/adapter/tarfs.py | 42 -------------------------------- moban/adapter/zipfs.py | 28 ---------------------- moban/fs_openers.py | 54 ++---------------------------------------- moban/utils.py | 4 ++-- 5 files changed, 4 insertions(+), 150 deletions(-) delete mode 100644 moban/adapter/osfs.py delete mode 100644 moban/adapter/tarfs.py delete mode 100644 moban/adapter/zipfs.py diff --git a/moban/adapter/osfs.py b/moban/adapter/osfs.py deleted file mode 100644 index ae637bb6..00000000 --- a/moban/adapter/osfs.py +++ /dev/null @@ -1,26 +0,0 @@ -import platform -import fs -from fs.osfs import OSFS -try: - from urllib.parse import quote -except ImportError: - from urllib import quote - -_WINDOWS_PLATFORM = platform.system() == "Windows" - - -class EnhancedOSFS(OSFS): - def geturl(self, path, purpose="download"): - if purpose != "download": - raise fs.errors.NoURL(path, purpose) - - sys_path = self.getsyspath(path) - if _WINDOWS_PLATFORM and ':' in sys_path: - drive_letter, path = sys_path.split(':', 1) - path = path.replace("\\", "/") - path = quote(path) - url_path = "{}:{}".format(drive_letter, path) - else: - sys_path = sys_path.replace("\\", "/") - url_path = quote(sys_path) - return "file://" + url_path diff --git a/moban/adapter/tarfs.py b/moban/adapter/tarfs.py deleted file mode 100644 index 0174b990..00000000 --- a/moban/adapter/tarfs.py +++ /dev/null @@ -1,42 +0,0 @@ -import os - -import six -from fs.tarfs import TarFS, ReadTarFS, WriteTarFS - - -class EnhancedReadTarFS(ReadTarFS): - def geturl(self, path, purpose="download"): - return "tar://%s!/%s" % (self._file, path) - - -class EnhancedTarFS(TarFS): - def __new__( - cls, - file, - write=False, - compression=None, - encoding="utf-8", - temp_fs="temp://__tartemp__", - ): - if isinstance(file, (six.text_type, six.binary_type)): - file = os.path.expanduser(file) - filename = file - else: - filename = getattr(file, "name", "") - - if write and compression is None: - compression = None - for comp, extensions in six.iteritems(cls._compression_formats): - if filename.endswith(extensions): - compression = comp - break - - if write: - return WriteTarFS( - file, - compression=compression, - encoding=encoding, - temp_fs=temp_fs, - ) - else: - return EnhancedReadTarFS(file, encoding=encoding) diff --git a/moban/adapter/zipfs.py b/moban/adapter/zipfs.py deleted file mode 100644 index a9050275..00000000 --- a/moban/adapter/zipfs.py +++ /dev/null @@ -1,28 +0,0 @@ -import zipfile - -from fs.zipfs import ZipFS, ReadZipFS, WriteZipFS - - -class EnhancedReadZipFS(ReadZipFS): - def geturl(self, path, purpose="download"): - return "zip://%s!/%s" % (self._file, path) - - -class EnhancedZipFS(ZipFS): - def __new__( - cls, - file, - write=False, - compression=zipfile.ZIP_DEFLATED, - encoding="utf-8", - temp_fs="temp://__ziptemp__", - ): - if write: - return WriteZipFS( - file, - compression=compression, - encoding=encoding, - temp_fs=temp_fs, - ) - else: - return EnhancedReadZipFS(file, encoding=encoding) diff --git a/moban/fs_openers.py b/moban/fs_openers.py index d31ab296..ae856d50 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -1,12 +1,10 @@ from moban import repo -from moban.adapter.osfs import EnhancedOSFS -from moban.adapter.tarfs import EnhancedTarFS -from moban.adapter.zipfs import EnhancedZipFS import fs import fs.path from fs.opener import Opener from fs.opener.registry import registry +from fs.osfs import OSFS @registry.install @@ -17,53 +15,5 @@ def open_fs(self, fs_url, parse_result, writeable, create, cwd): repo_name, _, dir_path = parse_result.resource.partition("/") actual_repo_path = fs.path.join(repo.get_moban_home(), repo_name) root_path = fs.path.join(actual_repo_path, dir_path) - osfs = EnhancedOSFS(root_path=root_path) - return osfs - - -@registry.install -class ZipOpener(Opener): - """`ZipFS` opener. - """ - - protocols = ["zip"] - - def open_fs(self, fs_url, parse_result, writeable, create, cwd): - if not create and writeable: - raise fs.errors.NotWriteable( - "Unable to open existing ZIP file for writing" - ) - zip_fs = EnhancedZipFS(parse_result.resource, write=create) - return zip_fs - - -@registry.install -class TarOpener(Opener): - """`TarFS` opener. - """ - - protocols = ["tar"] - - def open_fs(self, fs_url, parse_result, writeable, create, cwd): - if not create and writeable: - raise fs.errors.NotWriteable( - "Unable to open existing TAR file for writing" - ) - tar_fs = EnhancedTarFS(parse_result.resource, write=create) - return tar_fs - - -@registry.install -class OSFSOpener(Opener): - """`OSFS` opener. - """ - - protocols = ["file", "osfs"] - - def open_fs(self, fs_url, parse_result, writeable, create, cwd): - from os.path import abspath, expanduser, normpath, join - - _path = abspath(join(cwd, expanduser(parse_result.resource))) - path = normpath(_path) - osfs = EnhancedOSFS(path, create=create) + osfs = OSFS(root_path=root_path) return osfs diff --git a/moban/utils.py b/moban/utils.py index 4a6c1cc3..7213d5e3 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -39,7 +39,7 @@ def search_file(base_dir, file_name): try: with file_system.open_fs(base_dir) as fs_handle: if fs_handle.exists(file_system.to_unicode(the_file)): - the_file = fs_handle.geturl(file_name) + the_file = fs_handle.geturl(file_name, purpose='fs') else: raise IOError( constants.ERROR_DATA_FILE_NOT_FOUND @@ -113,7 +113,7 @@ def get_template_path(template_dirs, template): ) and fs_handle.isfile(template) if template_file_exists: - return fs_handle.geturl(template) + return fs_handle.geturl(template, purpose='fs') except fs.errors.CreateFailed: continue raise exceptions.FileNotFound From 85ea8acd4d2a8ec274a0d00bbd986debd7e7dd28 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 7 Aug 2019 07:52:57 +0100 Subject: [PATCH 109/167] :ambulance: use next version of jinja2-fsloader. https://github.com/althonos/jinja2-fsloader/pull/3 --- rnd_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/rnd_requirements.txt b/rnd_requirements.txt index efab09e1..04f0d6e8 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,4 +1,5 @@ https://github.com/moremoban/pyfilesystem2/archive/master.zip +https://github.com/moremoban/jinja2-fsloader/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip https://github.com/moremoban/pypifs/archive/master.zip From 91198d1786e1fcaa2bf42a5647e6695720b42984 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 7 Aug 2019 08:40:09 +0100 Subject: [PATCH 110/167] :fire: temporarily disable min requirement test beacause min requirement, to me, cannot point to un-released version of pyfs2, jinja2-fsloader --- .moban.d/moban_travis.yml.jj2 | 1 - .travis.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.moban.d/moban_travis.yml.jj2 b/.moban.d/moban_travis.yml.jj2 index 91ba2fd4..dc4c5c83 100644 --- a/.moban.d/moban_travis.yml.jj2 +++ b/.moban.d/moban_travis.yml.jj2 @@ -3,7 +3,6 @@ {%block extra_matrix %} env: - MINREQ=0 - - MINREQ=1 {%endblock%} {%block custom_python_versions%} diff --git a/.travis.yml b/.travis.yml index 4f34a3e4..d28e28d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,6 @@ python: - 3.8-dev env: - MINREQ=0 - - MINREQ=1 stages: - test From 485e4f211d73c49ecdf3c06bc127b44b0879f982 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 7 Aug 2019 19:25:01 +0100 Subject: [PATCH 111/167] :shirt: code formatting --- .isort.cfg | 2 +- moban/buffered_writer.py | 3 +-- moban/fs_openers.py | 5 ++--- moban/jinja2/engine.py | 5 +++-- moban/utils.py | 7 +++---- tests/mobanfile/test_mobanfile.py | 3 +-- tests/mobanfile/test_targets.py | 3 +-- tests/mobanfile/test_templates.py | 3 +-- tests/test_context.py | 3 +-- tests/test_copy_encoding.py | 3 +-- tests/test_docs.py | 1 - tests/test_engine.py | 3 +-- tests/test_json_loader.py | 3 +-- tests/test_regression.py | 3 +-- tests/test_repo.py | 3 +-- tests/test_template.py | 3 +-- tests/test_utils.py | 3 +-- tests/test_yaml_loader.py | 3 +-- 18 files changed, 22 insertions(+), 37 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index d76a1394..25965d9b 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -2,7 +2,7 @@ line_length=79 # Ignore generated files skip=setup.py, moban/__init__.py -known_first_party=lml, crayons, jinja2, ruamel.yaml, mock, nose +known_first_party=fs, lml, crayons, jinja2, ruamel.yaml, mock, nose indent=' ' multi_line_output=3 length_sort=1 diff --git a/moban/buffered_writer.py b/moban/buffered_writer.py index 49cbdb3e..90c453b8 100644 --- a/moban/buffered_writer.py +++ b/moban/buffered_writer.py @@ -1,7 +1,6 @@ -from moban import utils, file_system - import fs import fs.path +from moban import utils, file_system class BufferedWriter(object): diff --git a/moban/fs_openers.py b/moban/fs_openers.py index ae856d50..f22512c4 100644 --- a/moban/fs_openers.py +++ b/moban/fs_openers.py @@ -1,10 +1,9 @@ -from moban import repo - import fs import fs.path +from moban import repo +from fs.osfs import OSFS from fs.opener import Opener from fs.opener.registry import registry -from fs.osfs import OSFS @registry.install diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index e82acfc4..3de3bf69 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -3,14 +3,15 @@ from importlib import import_module import fs -from fs.multifs import MultiFS from moban import constants, exceptions from jinja2 import Template, Environment +from fs.multifs import MultiFS from lml.loader import scan_plugins_regex from lml.plugin import PluginInfo, PluginManager -from jinja2_fsloader import FSLoader from jinja2.exceptions import TemplateNotFound +from jinja2_fsloader import FSLoader + JINJA2_LIBRARIES = "^moban_jinja2_.+$" JINJA2_EXENSIONS = [ "moban.jinja2.filters.repr", diff --git a/moban/utils.py b/moban/utils.py index 7213d5e3..bd3dfe5d 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -4,9 +4,8 @@ import errno import logging -from moban import constants, exceptions, file_system - import fs +from moban import constants, exceptions, file_system LOG = logging.getLogger(__name__) PY2 = sys.version_info[0] == 2 @@ -39,7 +38,7 @@ def search_file(base_dir, file_name): try: with file_system.open_fs(base_dir) as fs_handle: if fs_handle.exists(file_system.to_unicode(the_file)): - the_file = fs_handle.geturl(file_name, purpose='fs') + the_file = fs_handle.geturl(file_name, purpose="fs") else: raise IOError( constants.ERROR_DATA_FILE_NOT_FOUND @@ -113,7 +112,7 @@ def get_template_path(template_dirs, template): ) and fs_handle.isfile(template) if template_file_exists: - return fs_handle.geturl(template, purpose='fs') + return fs_handle.geturl(template, purpose="fs") except fs.errors.CreateFailed: continue raise exceptions.FileNotFound diff --git a/tests/mobanfile/test_mobanfile.py b/tests/mobanfile/test_mobanfile.py index 321b8e2f..1eb3143e 100644 --- a/tests/mobanfile/test_mobanfile.py +++ b/tests/mobanfile/test_mobanfile.py @@ -1,9 +1,8 @@ +import fs.path from mock import patch from nose.tools import eq_ from moban.definitions import GitRequire, TemplateTarget -import fs.path - class TestFinder: def setUp(self): diff --git a/tests/mobanfile/test_targets.py b/tests/mobanfile/test_targets.py index 18b7b93a..dff5c850 100644 --- a/tests/mobanfile/test_targets.py +++ b/tests/mobanfile/test_targets.py @@ -1,12 +1,11 @@ import uuid +import fs.path from nose.tools import eq_, raises from moban.mobanfile import targets from moban.exceptions import GroupTargetNotFound from moban.definitions import TemplateTarget -import fs.path - TEMPLATE = "a.jj2" OUTPUT = "a.output" CONFIGURATION = "data.config" diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index 61b69479..7b362d10 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -1,9 +1,8 @@ +import fs.path from mock import patch from nose.tools import eq_ from moban.mobanfile.templates import handle_template -import fs.path - class TestHandleTemplateFunction: def setUp(self): diff --git a/tests/test_context.py b/tests/test_context.py index e74019c9..72575582 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,10 +1,9 @@ import os +import fs.path from nose.tools import eq_ from moban.plugins.context import Context -import fs.path - def test_context(): context = Context(fs.path.join("tests", "fixtures")) diff --git a/tests/test_copy_encoding.py b/tests/test_copy_encoding.py index 50880ef8..bcad3674 100644 --- a/tests/test_copy_encoding.py +++ b/tests/test_copy_encoding.py @@ -1,8 +1,7 @@ +import fs.path from moban.copy import ContentForwardEngine from nose.tools import eq_ -import fs.path - class TestCopyEncoding: def setUp(self): diff --git a/tests/test_docs.py b/tests/test_docs.py index 003722bb..f025d647 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -6,7 +6,6 @@ from moban import file_system from moban.main import main from nose.tools import eq_ - from fs.opener.parse import parse_fs_url diff --git a/tests/test_engine.py b/tests/test_engine.py index 114bac97..0e417e14 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,6 +1,7 @@ import os import sys +import fs.path import moban.exceptions as exceptions from mock import patch from lml.plugin import PluginInfo @@ -14,8 +15,6 @@ from moban.plugins.context import Context from moban.plugins.template import MobanEngine, expand_template_directories -import fs.path - USER_HOME = fs.path.join("user", "home", ".moban", "repos") diff --git a/tests/test_json_loader.py b/tests/test_json_loader.py index 39c00e77..44ed74fa 100644 --- a/tests/test_json_loader.py +++ b/tests/test_json_loader.py @@ -1,8 +1,7 @@ +import fs.path from nose.tools import eq_ from moban.data_loaders.json_loader import open_json -import fs.path - def test_open_json(): content = open_json(fs.path.join("tests", "fixtures", "child.json")) diff --git a/tests/test_regression.py b/tests/test_regression.py index 894267e7..37b771d4 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -3,12 +3,11 @@ import filecmp from textwrap import dedent +import fs.path from mock import patch from moban.main import main from nose.tools import eq_ -import fs.path - def custom_dedent(long_texts): refined = dedent(long_texts) diff --git a/tests/test_repo.py b/tests/test_repo.py index 803e7a82..0041cc4e 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,3 +1,4 @@ +import fs.path from mock import patch from moban.repo import ( git_clone, @@ -9,8 +10,6 @@ from moban.exceptions import NoGitCommand from moban.definitions import GitRequire -import fs.path - @patch("appdirs.user_cache_dir", return_value="root") @patch("moban.utils.mkdir_p") diff --git a/tests/test_template.py b/tests/test_template.py index f4184432..091f9f0c 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -1,5 +1,6 @@ import os +import fs.path from mock import patch from nose.tools import eq_ from moban.plugins import ENGINES @@ -7,8 +8,6 @@ from moban.jinja2.engine import Engine from moban.data_loaders.yaml import open_yaml -import fs.path - MODULE = "moban.plugins.template" diff --git a/tests/test_utils.py b/tests/test_utils.py index 4e999af4..535db5d2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,6 +3,7 @@ import stat from shutil import rmtree +import fs.path from mock import patch from nose import SkipTest from nose.tools import eq_, raises @@ -15,8 +16,6 @@ ) from moban.exceptions import FileNotFound -import fs.path - def create_file(test_file, permission): with open(test_file, "w") as f: diff --git a/tests/test_yaml_loader.py b/tests/test_yaml_loader.py index ce453c8b..c8f2c8b1 100644 --- a/tests/test_yaml_loader.py +++ b/tests/test_yaml_loader.py @@ -1,9 +1,8 @@ +import fs.path from nose.tools import eq_, raises from moban.data_loaders.yaml import open_yaml from moban.data_loaders.manager import load_data -import fs.path - def test_simple_yaml(): test_file = fs.path.join("tests", "fixtures", "simple.yaml") From 525c68b9e5a30d7e7377a4a439e8fc5e0566934c Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 7 Aug 2019 19:27:09 +0100 Subject: [PATCH 112/167] :shirt: use released version jinja2-fsloader. https://github.com/althonos/jinja2-fsloader/pull/3 --- .moban.cd/moban.yml | 2 +- min_requirements.txt | 2 +- requirements.txt | 2 +- rnd_requirements.txt | 1 - setup.py | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 16e7e55c..73e1b50c 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -29,7 +29,7 @@ dependencies: - GitPython>=2.0.0 - git-url-parse>=1.2.2 - fs>=2.4.6 - - jinja2-fsloader>=0.1.3 + - jinja2-fsloader>=0.2.0 description: Yet another jinja2 cli command for static text generation scm_host: github.com lint_command: make install_test format git-diff-check lint diff --git a/min_requirements.txt b/min_requirements.txt index 6921d623..802131e9 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -9,4 +9,4 @@ crayons== 0.1.0 GitPython==2.0.0 git-url-parse==1.2.2 fs==2.4.6 -jinja2-fsloader==0.1.3 +jinja2-fsloader==0.2.0 diff --git a/requirements.txt b/requirements.txt index 7e261a09..b8b77a79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ crayons>= 0.1.0 GitPython>=2.0.0 git-url-parse>=1.2.2 fs>=2.4.6 -jinja2-fsloader>=0.1.3 +jinja2-fsloader>=0.2.0 diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 04f0d6e8..efab09e1 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,5 +1,4 @@ https://github.com/moremoban/pyfilesystem2/archive/master.zip -https://github.com/moremoban/jinja2-fsloader/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip https://github.com/moremoban/pypifs/archive/master.zip diff --git a/setup.py b/setup.py index 46a52b34..33335d93 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ "GitPython>=2.0.0", "git-url-parse>=1.2.2", "fs>=2.4.6", - "jinja2-fsloader>=0.1.3", + "jinja2-fsloader>=0.2.0", ] SETUP_COMMANDS = {} From eaf5d7b8937b37764880aab7264f0ba215bf231a Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 8 Aug 2019 07:49:58 +0100 Subject: [PATCH 113/167] :hammer: code refactoring --- moban/file_system.py | 12 ++++++++ moban/mobanfile/templates.py | 53 ++++++++++-------------------------- 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index da0bcfa0..8da196f0 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -18,6 +18,18 @@ path_splitext = fs.path.splitext +def url_join(path, path2): + return zip_alike_url_join(path, path2) + + +def zip_alike_url_join(path, path2): + result = urlparse(path) + if path.endswith(result.scheme): + return path + to_unicode("!/") + path2 + else: + return path + to_unicode("/") + path2 + + def log_fs_failure(function_in_this_module): def wrapper(*args, **kwds): try: diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 29e1376b..5b133f30 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -37,52 +37,27 @@ def handle_template(template_file, output, template_dirs): "{0} cannot be found".format(template_file) ) elif file_system.is_dir(template_file_on_disk): - if "zip://" in template_file_on_disk: - for a_triple in _list_zip_dir_files( - template_file, template_file_on_disk, output - ): - yield a_triple - else: - for a_triple in _list_dir_files( - template_file, template_file_on_disk, output - ): - yield a_triple + for a_triple in _list_dir_files( + template_file, template_file_on_disk, output + ): + yield a_triple else: template_type = _get_template_type(template_file) yield (template_file, output, template_type) def _list_dir_files(source, actual_source_path, dest): - with file_system.open_fs(actual_source_path) as fs_handle: - for file_name in fs_handle.listdir(u"."): - if fs_handle.isfile(file_system.to_unicode(file_name)): - # please note jinja2 does NOT like windows path - # hence the following statement looks like cross platform - # src_file_under_dir = os.path.join(source, file_name) - # but actually it breaks windows instead. - src_file_under_dir = "%s/%s" % (source, file_name) - - dest_file_under_dir = file_system.path_join(dest, file_name) - template_type = _get_template_type(src_file_under_dir) - yield (src_file_under_dir, dest_file_under_dir, template_type) + for file_name in file_system.list_dir(actual_source_path): + if file_system.is_file(file_system.url_join(actual_source_path, file_name)): + # please note jinja2 does NOT like windows path + # hence the following statement looks like cross platform + # src_file_under_dir = os.path.join(source, file_name) + # but actually it breaks windows instead. + src_file_under_dir = "%s/%s" % (source, file_name) - -def _list_zip_dir_files(source, actual_source_path, dest): - zip_file, folder = file_system.url_split(actual_source_path) - with file_system.open_fs(zip_file) as fs_handle: - for file_name in fs_handle.listdir(file_system.to_unicode(folder)): - if fs_handle.isfile( - file_system.to_unicode(folder + "/" + file_name) - ): - # please note jinja2 does NOT like windows path - # hence the following statement looks like cross platform - # src_file_under_dir = os.path.join(source, file_name) - # but actually it breaks windows instead. - src_file_under_dir = "%s/%s" % (source, file_name) - - dest_file_under_dir = dest + "/" + file_name - template_type = _get_template_type(src_file_under_dir) - yield (src_file_under_dir, dest_file_under_dir, template_type) + dest_file_under_dir = dest + "/" + file_name + template_type = _get_template_type(src_file_under_dir) + yield (src_file_under_dir, dest_file_under_dir, template_type) def _listing_directory_files_recusively(source, actual_source_path, dest): From 641b5a0f7fc61b8b174313f4005917f32a2cb7dc Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 8 Aug 2019 13:23:09 +0100 Subject: [PATCH 114/167] :tractor: switch the lane from file:// to osfs://. https://github.com/PyFilesystem/pyfilesystem2/pull/330 --- tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 535db5d2..66184b8a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -112,7 +112,7 @@ def test_get_template_path(): ] template = "a.jj2" template_path = get_template_path(temp_dirs, template) - expected = "file://" + os.path.normcase( + expected = "osfs://" + os.path.normcase( os.path.abspath( os.path.join("tests", "fixtures", "template-tests", "a.jj2") ) From 4aec26297bd24d4da1fda69d3ac4594e73a6a376 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 8 Aug 2019 20:38:48 +0100 Subject: [PATCH 115/167] :shirt: update coding style --- docs/level-10-moban-dependency-as-git-repo/.moban.yml | 4 ++-- moban/mobanfile/templates.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/level-10-moban-dependency-as-git-repo/.moban.yml b/docs/level-10-moban-dependency-as-git-repo/.moban.yml index 884a8206..114d6027 100644 --- a/docs/level-10-moban-dependency-as-git-repo/.moban.yml +++ b/docs/level-10-moban-dependency-as-git-repo/.moban.yml @@ -2,10 +2,10 @@ requires: - https://github.com/moremoban/pypi-mobans configuration: template_dir: - - "repo://pypi-mobans/templates" + - "usercache://moban:chfw/repos/pypi-mobans/templates" - local configuration: config.yml - configuration_dir: "repo://pypi-mobans/config" + configuration_dir: "usercache://moban:chfw/repos/pypi-mobans/config" targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 5b133f30..b62ff437 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -48,7 +48,9 @@ def handle_template(template_file, output, template_dirs): def _list_dir_files(source, actual_source_path, dest): for file_name in file_system.list_dir(actual_source_path): - if file_system.is_file(file_system.url_join(actual_source_path, file_name)): + if file_system.is_file( + file_system.url_join(actual_source_path, file_name) + ): # please note jinja2 does NOT like windows path # hence the following statement looks like cross platform # src_file_under_dir = os.path.join(source, file_name) From 73155ccef73bed1ed86b4a32044ad6065234a337 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 8 Aug 2019 21:38:49 +0100 Subject: [PATCH 116/167] :green_heart: revert unwanted change and make windows unit tests pass --- docs/level-10-moban-dependency-as-git-repo/.moban.yml | 4 ++-- moban/file_system.py | 2 +- tests/test_file_system.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/level-10-moban-dependency-as-git-repo/.moban.yml b/docs/level-10-moban-dependency-as-git-repo/.moban.yml index 114d6027..884a8206 100644 --- a/docs/level-10-moban-dependency-as-git-repo/.moban.yml +++ b/docs/level-10-moban-dependency-as-git-repo/.moban.yml @@ -2,10 +2,10 @@ requires: - https://github.com/moremoban/pypi-mobans configuration: template_dir: - - "usercache://moban:chfw/repos/pypi-mobans/templates" + - "repo://pypi-mobans/templates" - local configuration: config.yml - configuration_dir: "usercache://moban:chfw/repos/pypi-mobans/config" + configuration_dir: "repo://pypi-mobans/config" targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 diff --git a/moban/file_system.py b/moban/file_system.py index 8da196f0..5dbdba62 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -188,7 +188,7 @@ def fs_url(path): path = to_unicode(path) folder_or_file, path = _path_split(path) with fs.open_fs(folder_or_file) as the_fs: - return the_fs.geturl(path) + return the_fs.geturl(path, purpose="fs") def to_unicode(path): diff --git a/tests/test_file_system.py b/tests/test_file_system.py index e41fc189..fca70eac 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -159,7 +159,7 @@ def test_abspath(): TEST_FILE_URL = [ [ LOCAL_FOLDER + "/file_system", - "file://" + "osfs://" + os.path.normpath( os.path.join(os.getcwd(), "tests/fixtures/file_system") ), From 22ec2ba3363539df3ba0456a8b045fea5a28c3d0 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 8 Aug 2019 22:32:04 +0100 Subject: [PATCH 117/167] :fire: remove duplicated codes --- moban/file_system.py | 2 +- moban/mobanfile/templates.py | 60 ++++++++----------------------- tests/mobanfile/test_templates.py | 2 +- 3 files changed, 17 insertions(+), 47 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index 5dbdba62..40fda162 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -24,7 +24,7 @@ def url_join(path, path2): def zip_alike_url_join(path, path2): result = urlparse(path) - if path.endswith(result.scheme): + if result.scheme and path.endswith(result.scheme): return path + to_unicode("!/") + path2 else: return path + to_unicode("/") + path2 diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index b62ff437..b12b24a5 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -13,16 +13,10 @@ def handle_template(template_file, output, template_dirs): source_dir = template_file[:-3] src_path = find_file_in_template_dirs(source_dir, template_dirs) if src_path: - if "zip://" in src_path: - for a_triple in _listing_zip_directory_files_recusively( - source_dir, src_path, output - ): - yield a_triple - else: - for a_triple in _listing_directory_files_recusively( - source_dir, src_path, output - ): - yield a_triple + for a_triple in _listing_directory_files_recusively( + source_dir, src_path, output + ): + yield a_triple else: reporter.report_error_message( "{0} cannot be found".format(template_file) @@ -63,42 +57,18 @@ def _list_dir_files(source, actual_source_path, dest): def _listing_directory_files_recusively(source, actual_source_path, dest): - with file_system.open_fs(actual_source_path) as fs_handle: - for file_name in fs_handle.listdir(u"."): - src_file_under_dir = file_system.path_join(source, file_name) - dest_file_under_dir = file_system.path_join(dest, file_name) - real_src_file = "%s/%s" % (actual_source_path, file_name) - if fs_handle.isfile(file_system.to_unicode(file_name)): - template_type = _get_template_type(src_file_under_dir) - yield (src_file_under_dir, dest_file_under_dir, template_type) - elif fs_handle.isdir(file_system.to_unicode(file_name)): - for a_triple in _listing_directory_files_recusively( - src_file_under_dir, real_src_file, dest_file_under_dir - ): - yield a_triple - - -def _listing_zip_directory_files_recusively(source, actual_source_path, dest): - zip_file, folder = file_system.url_split(actual_source_path) - print(actual_source_path) - - with file_system.open_fs(zip_file) as fs_handle: - for file_name in fs_handle.listdir(file_system.to_unicode(folder)): - src_file_under_dir = source + "/" + file_name - dest_file_under_dir = dest + "/" + file_name - real_src_file = "%s/%s" % (actual_source_path, file_name) - if fs_handle.isfile( - file_system.to_unicode(folder + "/" + file_name) - ): - template_type = _get_template_type(src_file_under_dir) - yield (src_file_under_dir, dest_file_under_dir, template_type) - elif fs_handle.isdir( - file_system.to_unicode(folder + "/" + file_name) + for file_name in file_system.list_dir(actual_source_path): + src_file_under_dir = source + "/" + file_name + dest_file_under_dir = dest + "/" + file_name + real_src_file = file_system.url_join(actual_source_path, file_name) + if file_system.is_file(real_src_file): + template_type = _get_template_type(src_file_under_dir) + yield (src_file_under_dir, dest_file_under_dir, template_type) + elif file_system.is_dir(real_src_file): + for a_triple in _listing_directory_files_recusively( + src_file_under_dir, real_src_file, dest_file_under_dir ): - for a_triple in _listing_zip_directory_files_recusively( - src_file_under_dir, real_src_file, dest_file_under_dir - ): - yield a_triple + yield a_triple def _get_template_type(template_file): diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index 7b362d10..bd07f6ff 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -60,8 +60,8 @@ def test_listing_dir_recusively(self): ), ] eq_( - sorted(expected, key=lambda x: x[0]), sorted(results, key=lambda x: x[0]), + sorted(expected, key=lambda x: x[0]), ) @patch("moban.reporter.report_error_message") From 609c43544f8877af5220d631d9ed70ee69c22e98 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 8 Aug 2019 23:33:04 +0100 Subject: [PATCH 118/167] :hammer: replace custom join with url_join --- moban/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index bd3dfe5d..52d41bf5 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -147,9 +147,7 @@ def verify_the_existence_of_directories(dirs): def find_file_in_template_dirs(src, template_dirs): LOG.debug(template_dirs) for folder in template_dirs: - path = folder + "/" + src - if "zip://" in folder or "tar://" in folder: - path = folder + "!/" + src + path = file_system.url_join(folder, src) if file_system.exists(path): return path else: From d38a5f414a99d932fb04b415902c8d78fa113a7c Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 8 Aug 2019 23:50:42 +0100 Subject: [PATCH 119/167] :hammer: optimize the code --- moban/utils.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 52d41bf5..4bb9d63d 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -35,16 +35,10 @@ def search_file(base_dir, file_name): the_file = file_name if not file_system.exists(the_file): if base_dir: - try: - with file_system.open_fs(base_dir) as fs_handle: - if fs_handle.exists(file_system.to_unicode(the_file)): - the_file = fs_handle.geturl(file_name, purpose="fs") - else: - raise IOError( - constants.ERROR_DATA_FILE_NOT_FOUND - % (file_name, the_file) - ) - except fs.errors.CreateFailed: + file_under_base_dir = file_system.url_join(base_dir, the_file) + if file_system.exists(file_under_base_dir): + the_file = file_system.fs_url(file_under_base_dir) + else: raise IOError( constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) ) From 7061996b40d9e4f609a0d44a431f69389c154359 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Aug 2019 07:26:53 +0100 Subject: [PATCH 120/167] :hammer: minor update on checking filename --- moban/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/utils.py b/moban/utils.py index 4bb9d63d..b685a4ae 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -69,7 +69,7 @@ def write_file_out(filename, content): if PY2 and content.__class__.__name__ == "unicode": content = content.encode("utf-8") - if "zip://" not in filename and "tar://" not in filename: + if file_system.is_zip_alike_url(filename): # fix me dest_folder = os.path.dirname(filename) if dest_folder: From ad459b679ffcc040149553e9512e4381946a22e2 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Aug 2019 07:27:07 +0100 Subject: [PATCH 121/167] :books: update yaml indentation --- .../README.rst | 6 +++--- docs/level-10-moban-dependency-as-git-repo/README.rst | 2 +- docs/level-9-moban-dependency-as-pypi-package/README.rst | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/deprecated-level-9-moban-dependency-as-pypi-package/README.rst b/docs/deprecated-level-9-moban-dependency-as-pypi-package/README.rst index 7bc94496..1366bbc6 100644 --- a/docs/deprecated-level-9-moban-dependency-as-pypi-package/README.rst +++ b/docs/deprecated-level-9-moban-dependency-as-pypi-package/README.rst @@ -12,7 +12,7 @@ into a pypi package and distribute it to the world of moban. Here are the sample file:: requires: - - pypi-mobans + - pypi-mobans configuration: template_dir: - "pypi-mobans:templates" @@ -44,6 +44,6 @@ Alternative syntax The alternative syntax is:: requires: - - type: pypi - name: pypi-mobans + - type: pypi + name: pypi-mobans ... diff --git a/docs/level-10-moban-dependency-as-git-repo/README.rst b/docs/level-10-moban-dependency-as-git-repo/README.rst index 61de19db..32e0f568 100644 --- a/docs/level-10-moban-dependency-as-git-repo/README.rst +++ b/docs/level-10-moban-dependency-as-git-repo/README.rst @@ -12,7 +12,7 @@ is welcome to add or improve this feature. Here are the sample file:: requires: - - https://github.com/moremoban/pypi-mobans + - https://github.com/moremoban/pypi-mobans configuration: template_dir: - "repo://pypi-mobans/templates" diff --git a/docs/level-9-moban-dependency-as-pypi-package/README.rst b/docs/level-9-moban-dependency-as-pypi-package/README.rst index ebf4e913..14ccb3c0 100644 --- a/docs/level-9-moban-dependency-as-pypi-package/README.rst +++ b/docs/level-9-moban-dependency-as-pypi-package/README.rst @@ -45,6 +45,6 @@ Alternative syntax The alternative syntax is:: requires: - - type: pypi - name: pypi-mobans + - type: pypi + name: pypi-mobans ... From 28a6b80ddf1d20edf4b26157053b69b35d64e8a0 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Aug 2019 07:35:10 +0100 Subject: [PATCH 122/167] :green_heart: :hammer: code refactor utils.py --- moban/utils.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index b685a4ae..a3c35aeb 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -69,7 +69,7 @@ def write_file_out(filename, content): if PY2 and content.__class__.__name__ == "unicode": content = content.encode("utf-8") - if file_system.is_zip_alike_url(filename): + if not file_system.is_zip_alike_url(filename): # fix me dest_folder = os.path.dirname(filename) if dest_folder: @@ -99,14 +99,13 @@ def pip_install(packages): def get_template_path(template_dirs, template): for a_dir in template_dirs: try: - with file_system.open_fs(a_dir) as fs_handle: - template = file_system.to_unicode(template) - template_file_exists = fs_handle.exists( - template - ) and fs_handle.isfile(template) - - if template_file_exists: - return fs_handle.geturl(template, purpose="fs") + template_under_dir = file_system.url_join(a_dir, template) + template_file_exists = file_system.exists( + template_under_dir + ) and file_system.is_file(template_under_dir) + + if template_file_exists: + return file_system.fs_url(template_under_dir) except fs.errors.CreateFailed: continue raise exceptions.FileNotFound From 03acbc5c1203a536c4851b1d75530e2aa2f85d0b Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Aug 2019 07:46:55 +0100 Subject: [PATCH 123/167] :fire: remove unwanted, useless changes occured in development stage. now it is proven not helping at all. --- moban/definitions.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/moban/definitions.py b/moban/definitions.py index b2666add..9cd73a73 100644 --- a/moban/definitions.py +++ b/moban/definitions.py @@ -46,9 +46,9 @@ def __init__( output, template_type=constants.DEFAULT_TEMPLATE_TYPE, ): - self.template_file = to_unicode(template_file) - self.data_file = to_unicode(data_file) - self.original_output = to_unicode(output) + self.template_file = template_file + self.data_file = data_file + self.original_output = output self.template_type = template_type self.output = self.original_output @@ -56,7 +56,7 @@ def __init__( def set_template_type(self, new_template_type): self.template_type = new_template_type - if self.original_output.endswith(to_unicode(self.template_type)): + if self.original_output.endswith(self.template_type): self.output, _ = os.path.splitext(self.original_output) else: self.output = self.original_output @@ -76,9 +76,3 @@ def __repr__(self): self.output, self.template_type, ) - - -def to_unicode(path): - if PY2 and path.__class__.__name__ != "unicode": - return u"".__class__(path) - return path From e26c050c05b46d35b3fca5403b7865c0bbe2fd72 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Aug 2019 07:54:57 +0100 Subject: [PATCH 124/167] :fire: remove unwanted changes. --- moban/definitions.py | 3 --- moban/mobanfile/__init__.py | 2 -- moban/mobanfile/targets.py | 1 - moban/plugins/template.py | 1 - tests/test_engine.py | 2 -- 5 files changed, 9 deletions(-) diff --git a/moban/definitions.py b/moban/definitions.py index 9cd73a73..8ae4ea45 100644 --- a/moban/definitions.py +++ b/moban/definitions.py @@ -1,10 +1,7 @@ import os -import sys from moban import constants -PY2 = sys.version_info[0] == 2 - class GitRequire(object): def __init__( diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index 09d56aa2..197d1d3f 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -109,7 +109,6 @@ def _iterate_list_of_dicts(list_of_dict): def handle_targets(merged_options, targets): list_of_templating_parameters = parse_targets(merged_options, targets) - jobs_for_each_engine = OrderedDict() for target in list_of_templating_parameters: @@ -133,7 +132,6 @@ def handle_targets(merged_options, targets): jobs_for_each_engine[primary_template_type].append(target) count = 0 - for template_type in jobs_for_each_engine.keys(): engine = plugins.ENGINES.get_engine( template_type, diff --git a/moban/mobanfile/targets.py b/moban/mobanfile/targets.py index 07505df0..e32c8767 100644 --- a/moban/mobanfile/targets.py +++ b/moban/mobanfile/targets.py @@ -40,7 +40,6 @@ def parse_targets(options, targets): def _handle_explicit_target(options, target): - common_data_file = options[constants.LABEL_CONFIG] default_template_type = options[constants.LABEL_TEMPLATE_TYPE] template_file = target.get( diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 05db6540..4f59b250 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -192,7 +192,6 @@ def expand_template_directories(dirs): def expand_template_directory(directory): log.debug("Expanding %s..." % directory) - translated_directory = None if ":" in directory and directory[1] != ":" and "://" not in directory: translated_directory = deprecated_moban_path_notation(directory) diff --git a/tests/test_engine.py b/tests/test_engine.py index 0e417e14..e04dd546 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -22,14 +22,12 @@ class TestPypkg: def __init__(self): __package_path__ = os.path.normcase(os.path.dirname(__file__)) - print(__package_path__) self.resources_path = os.path.join(__package_path__, "fixtures") def test_expand_pypi_dir(): dirs = list(expand_template_directories("testmobans:template-tests")) for directory in dirs: - print(directory) assert os.path.exists(directory) From bc8ec03e6e18d13a1d989b89a70eb6f0dddaf287 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Aug 2019 09:03:48 +0100 Subject: [PATCH 125/167] :bug: fix the regression error where hash store is not disabled because it cannot find the file falsely --- moban/file_system.py | 20 ++++++++++++++++++++ moban/hashstore.py | 6 ++---- moban/plugins/template.py | 3 ++- moban/utils.py | 8 ++------ tests/test_docs.py | 3 +-- tests/test_utils.py | 2 +- 6 files changed, 28 insertions(+), 14 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index 40fda162..b82eb6fe 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -1,4 +1,6 @@ +import os import sys +import stat import logging from contextlib import contextmanager @@ -191,6 +193,14 @@ def fs_url(path): return the_fs.geturl(path, purpose="fs") +@log_fs_failure +def system_path(path): + path = to_unicode(path) + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + return the_fs.getsyspath(path) + + def to_unicode(path): if PY2 and path.__class__.__name__ != "unicode": return u"".__class__(path) @@ -206,6 +216,16 @@ def is_zip_alike_url(url): return False +def file_permissions(url): + if sys.platform == "win32": + return 0 + elif is_zip_alike_url(url): + return 755 + else: + unix_path = system_path(url) + return stat.S_IMODE(os.stat(unix_path).st_mode) + + def url_split(url): result = urlparse(url) diff --git a/moban/hashstore.py b/moban/hashstore.py index 25b72518..6adbd604 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -3,8 +3,7 @@ import json import hashlib -import moban.utils as utils -import moban.constants as constants +from moban import utils, constants, file_system PY2 = sys.version_info[0] == 2 @@ -57,8 +56,7 @@ def save_hashes(self): def get_file_hash(afile): - with open(afile, "rb") as handle: - content = handle.read() + content = file_system.read_bytes(afile) content = _mix(content, oct(utils.file_permissions(afile))) return get_hash(content) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 4f59b250..49699620 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -133,7 +133,8 @@ def apply_template(self, template_abs_path, template, data, output_file): if "zip://" not in output_file and "tar://" not in output_file: utils.file_permissions_copy(template_abs_path, output_file) return flag - except exceptions.FileNotFound: + except exceptions.FileNotFound as e: + log.exception(e) self.buffered_writer.write_file_out(output_file, rendered_content) return True diff --git a/moban/utils.py b/moban/utils.py index a3c35aeb..c7fabd84 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -56,13 +56,9 @@ def file_permissions_copy(source, dest): def file_permissions(afile): - if "zip://" in afile or "tar://" in afile: - return 755 - if not os.path.exists(afile): + if not file_system.exists(afile): raise exceptions.FileNotFound(afile) - if sys.platform == "win32": - return 0 - return stat.S_IMODE(os.stat(afile).st_mode) + return file_system.file_permissions(afile) def write_file_out(filename, content): diff --git a/tests/test_docs.py b/tests/test_docs.py index f025d647..377a4dbd 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -350,8 +350,7 @@ def _raw_moban_with_fs2(self, args, folder, criterias): os.unlink(result.resource) # delete the zip file def tearDown(self): - if os.path.exists(".moban.hashes"): - os.unlink(".moban.hashes") + os.unlink(".moban.hashes") os.chdir(self.current) diff --git a/tests/test_utils.py b/tests/test_utils.py index 66184b8a..c3704826 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -29,7 +29,7 @@ def test_file_permission_copy(): raise SkipTest("No actual chmod on windows") test_source = "test_file_permission_copy1" test_dest = "test_file_permission_copy2" - create_file(test_source, 0o046) + create_file(test_source, 0o755) create_file(test_dest, 0o646) file_permissions_copy(test_source, test_dest) eq_( From 8a57815242bca238e027ba1594ea2a2b55fd532a Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Aug 2019 18:46:57 +0100 Subject: [PATCH 126/167] :green_heart: update hash store unit test to be windows friendly --- tests/test_hash_store.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 2e22ef17..1edab33d 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -7,7 +7,9 @@ class TestHashStore: def setUp(self): - self.source_template = os.path.join("tests", "fixtures", "a.jj2") + self.source_template = os.path.normcase( + os.path.join("tests", "fixtures", "a.jj2") + ) self.fixture = ( "test.out", "test content".encode("utf-8"), From 4a55353f32947b5a70d6c6f0eefcdfedc71d8690 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Aug 2019 18:52:45 +0100 Subject: [PATCH 127/167] :green_heart: fix the path again --- tests/test_hash_store.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 1edab33d..6eb65637 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -2,13 +2,14 @@ import sys from nose import SkipTest +from mmoban import file_system from moban.hashstore import HashStore class TestHashStore: def setUp(self): - self.source_template = os.path.normcase( - os.path.join("tests", "fixtures", "a.jj2") + self.source_template = file_system.path_join( + "tests", "fixtures", "a.jj2" ) self.fixture = ( "test.out", From 3f57a3270451c763bf3c569c8333b63d5b8af005 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Aug 2019 19:04:25 +0100 Subject: [PATCH 128/167] :green_heart: fix the typo --- tests/test_hash_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 6eb65637..3a8d5ae9 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -2,7 +2,7 @@ import sys from nose import SkipTest -from mmoban import file_system +from moban import file_system from moban.hashstore import HashStore From 5a8281a2ca76ff59642bc7ea4eb6df75819b9bfe Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Aug 2019 19:10:07 +0100 Subject: [PATCH 129/167] :microscope: more tests for buffered writer --- moban/buffered_writer.py | 2 +- moban/file_system.py | 1 + moban/utils.py | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moban/buffered_writer.py b/moban/buffered_writer.py index 90c453b8..44b4e23f 100644 --- a/moban/buffered_writer.py +++ b/moban/buffered_writer.py @@ -8,7 +8,7 @@ def __init__(self): self.fs_list = {} def write_file_out(self, filename, content): - if "zip://" in filename: + if file_system.is_zip_alike_url(filename): self.write_file_out_to_zip(filename, content) else: utils.write_file_out(filename, content) diff --git a/moban/file_system.py b/moban/file_system.py index b82eb6fe..e97e20f0 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -115,6 +115,7 @@ def read_bytes(path): read_binary = read_bytes +read_text = read_unicode @log_fs_failure diff --git a/moban/utils.py b/moban/utils.py index c7fabd84..886f78b3 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,6 +1,5 @@ import os import sys -import stat import errno import logging From 9187852c06685f4ebe095dfe4479cffa4a68962d Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 17:55:04 +0100 Subject: [PATCH 130/167] :hammer: code refactoring --- moban/data_loaders/manager.py | 2 ++ moban/file_system.py | 4 ---- tests/test_file_system.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index 5ab9e9d3..2372cbff 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -1,5 +1,7 @@ from moban import utils, constants, file_system from lml.plugin import PluginManager +import moban.data_loaders.yaml # noqa: F401 +import moban.data_loaders.json_loader # noqa: F401 class AnyDataLoader(PluginManager): diff --git a/moban/file_system.py b/moban/file_system.py index e97e20f0..f21bc7b5 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -21,10 +21,6 @@ def url_join(path, path2): - return zip_alike_url_join(path, path2) - - -def zip_alike_url_join(path, path2): result = urlparse(path) if result.scheme and path.endswith(result.scheme): return path + to_unicode("!/") + path2 diff --git a/tests/test_file_system.py b/tests/test_file_system.py index fca70eac..772d63af 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -171,3 +171,16 @@ def test_fs_url(): for path, expected in TEST_FILE_URL: url = file_system.fs_url(path) eq_(url, expected.replace("\\", "/")) + + +URL_JOIN_TEST_FIXTURES = [ + ["parent", "child", "parent/child"], + ["zip://test.zip", "file", "zip://test.zip!/file"], + ["/root", "path", "/root/path"] +] + + +def test_url_join(): + for parent, child, expected_path in URL_JOIN_TEST_FIXTURES: + actual = file_system.url_join(parent, child) + eq_(actual, expected_path) From 483d11f64c375381319a96a82896f2f193bf7ac8 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 18:20:05 +0100 Subject: [PATCH 131/167] :shirt: update coding style --- moban/data_loaders/manager.py | 4 ++-- tests/test_file_system.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index 2372cbff..2ae7fa68 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -1,7 +1,7 @@ -from moban import utils, constants, file_system -from lml.plugin import PluginManager import moban.data_loaders.yaml # noqa: F401 import moban.data_loaders.json_loader # noqa: F401 +from moban import utils, constants, file_system +from lml.plugin import PluginManager class AnyDataLoader(PluginManager): diff --git a/tests/test_file_system.py b/tests/test_file_system.py index 772d63af..445710f8 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -176,7 +176,7 @@ def test_fs_url(): URL_JOIN_TEST_FIXTURES = [ ["parent", "child", "parent/child"], ["zip://test.zip", "file", "zip://test.zip!/file"], - ["/root", "path", "/root/path"] + ["/root", "path", "/root/path"], ] From 6f79615c6cf77e492c05a0267854323e851dbaab Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 18:25:23 +0100 Subject: [PATCH 132/167] :hammer: further integration with pyfs2 --- moban/mobanfile/__init__.py | 3 ++- tests/mobanfile/test_mobanfile.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index 197d1d3f..b309441c 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -11,6 +11,7 @@ from moban.definitions import GitRequire from moban.plugins.template import expand_template_directories from moban.mobanfile.targets import parse_targets, extract_group_targets +from moban import file_system try: from urllib.parse import urlparse @@ -23,7 +24,7 @@ def find_default_moban_file(): for moban_file in constants.DEFAULT_MOBAN_FILES: - if os.path.exists(moban_file): + if file_system.exists(moban_file): break else: moban_file = None diff --git a/tests/mobanfile/test_mobanfile.py b/tests/mobanfile/test_mobanfile.py index 1eb3143e..a6009438 100644 --- a/tests/mobanfile/test_mobanfile.py +++ b/tests/mobanfile/test_mobanfile.py @@ -6,7 +6,7 @@ class TestFinder: def setUp(self): - self.patcher = patch("os.path.exists") + self.patcher = patch("moban.file_system.exists") self.fake_file_existence = self.patcher.start() self.fake_file_existence.__name__ = "fake" self.fake_file_existence.__module__ = "fake" From dd51c3ca10b6f90e772bc30c09c03566b0799499 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 18:57:32 +0100 Subject: [PATCH 133/167] :shirt: update coding style --- moban/mobanfile/__init__.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index b309441c..b96d439d 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -3,7 +3,7 @@ import sys from collections import OrderedDict -from moban import plugins, reporter, constants +from moban import plugins, reporter, constants, file_system from lml.utils import do_import from moban.repo import git_clone from moban.utils import merge, pip_install @@ -11,7 +11,6 @@ from moban.definitions import GitRequire from moban.plugins.template import expand_template_directories from moban.mobanfile.targets import parse_targets, extract_group_targets -from moban import file_system try: from urllib.parse import urlparse @@ -189,16 +188,14 @@ def handle_requires(requires): if isinstance(require, dict): require_type = require.get(constants.REQUIRE_TYPE, "") if require_type.upper() == constants.GIT_REQUIRE: - git_repos.append( - GitRequire( - git_url=require.get(constants.GIT_URL), - branch=require.get(constants.GIT_BRANCH), - reference=require.get(constants.GIT_REFERENCE), - submodule=require.get( - constants.GIT_HAS_SUBMODULE, False - ), - ) + git_require = GitRequire( + git_url=require.get(constants.GIT_URL), + branch=require.get(constants.GIT_BRANCH), + reference=require.get(constants.GIT_REFERENCE), + submodule=require.get(constants.GIT_HAS_SUBMODULE, False), ) + + git_repos.append(git_require) elif require_type.upper() == constants.PYPI_REQUIRE: pypi_pkgs.append(require.get(constants.PYPI_PACKAGE_NAME)) else: From a8c647b2750d3e03142933ed4d6ce0514a69f22f Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 21:13:41 +0100 Subject: [PATCH 134/167] :hammer: minor code tuning --- moban/plugins/template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 49699620..da3fcffc 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -34,6 +34,7 @@ def get_engine(self, template_type, template_dirs, context_dirs): template_dirs = utils.verify_the_existence_of_directories( template_dirs ) + context_dirs = expand_template_directory(context_dirs) if template_type in self.options_registry: custom_engine_spec = self.options_registry[template_type] engine_cls = self.load_me_now( @@ -63,7 +64,6 @@ def raise_exception(self, key): class MobanEngine(object): def __init__(self, template_dirs, context_dirs, engine): - context_dirs = expand_template_directory(context_dirs) self.context = Context(context_dirs) self.template_dirs = template_dirs self.engine = engine @@ -130,7 +130,7 @@ def apply_template(self, template_abs_path, template, data, output_file): self.buffered_writer.write_file_out( output_file, rendered_content ) - if "zip://" not in output_file and "tar://" not in output_file: + if not file_system.is_zip_alike_url(output_file): utils.file_permissions_copy(template_abs_path, output_file) return flag except exceptions.FileNotFound as e: From 48578ac4e3695fa77b8ab189a246630b82355dd2 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 21:18:17 +0100 Subject: [PATCH 135/167] :hammer: code refactoring. one place to import fs --- moban/file_system.py | 8 ++++++++ moban/jinja2/engine.py | 9 ++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index f21bc7b5..42e8b921 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -6,6 +6,7 @@ import fs import fs.path +from fs.multifs import MultiFS try: from urllib.parse import urlparse @@ -241,3 +242,10 @@ def _path_split(url_or_path): return url_split(url_or_path) else: return fs.path.dirname(url_or_path), fs.path.basename(url_or_path) + + +def get_multi_fs(directories): + filesystem = MultiFS() + for directory in directories: + filesystem.add_fs(directory, fs.open_fs(directory)) + return filesystem diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index 3de3bf69..652c41ee 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -2,10 +2,8 @@ import logging from importlib import import_module -import fs -from moban import constants, exceptions +from moban import constants, exceptions, file_system from jinja2 import Template, Environment -from fs.multifs import MultiFS from lml.loader import scan_plugins_regex from lml.plugin import PluginInfo, PluginManager from jinja2.exceptions import TemplateNotFound @@ -71,10 +69,7 @@ def __init__(self, template_dirs, options=None): LOG.debug("Jinja template dirs: %s", template_dirs) load_jinja2_extensions() self.template_dirs = template_dirs - filesystem = MultiFS() - for template_dir in template_dirs: - filesystem.add_fs(template_dir, fs.open_fs(template_dir)) - + filesystem = file_system.get_multi_fs(template_dirs) template_loader = FSLoader(filesystem) env_params = dict( loader=template_loader, From 4958b6276159d5cafd185d716b87052bd4daf47a Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 21:21:31 +0100 Subject: [PATCH 136/167] :green_heart: make the tests pass --- moban/plugins/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index da3fcffc..935414d7 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -34,7 +34,6 @@ def get_engine(self, template_type, template_dirs, context_dirs): template_dirs = utils.verify_the_existence_of_directories( template_dirs ) - context_dirs = expand_template_directory(context_dirs) if template_type in self.options_registry: custom_engine_spec = self.options_registry[template_type] engine_cls = self.load_me_now( @@ -64,6 +63,7 @@ def raise_exception(self, key): class MobanEngine(object): def __init__(self, template_dirs, context_dirs, engine): + context_dirs = expand_template_directory(context_dirs) self.context = Context(context_dirs) self.template_dirs = template_dirs self.engine = engine From 7ab2d2983c9ad9024a45b8d56b811f669b82ef5f Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 21:27:06 +0100 Subject: [PATCH 137/167] :hammer: further collapse the utils.py code get_template_path --- moban/plugins/template.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 935414d7..bb2eede5 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -65,6 +65,7 @@ class MobanEngine(object): def __init__(self, template_dirs, context_dirs, engine): context_dirs = expand_template_directory(context_dirs) self.context = Context(context_dirs) + self.template_fs = file_system.get_multi_fs(template_dirs) self.template_dirs = template_dirs self.engine = engine self.templated_count = 0 @@ -88,8 +89,8 @@ def render_to_file(self, template_file, data_file, output_file): template = self.engine.get_template( file_system.to_unicode(template_file) ) - template_abs_path = utils.get_template_path( - self.template_dirs, template_file + template_abs_path = self.template_fs.geturl( + template_file, purpose="fs" ) flag = self.apply_template( @@ -152,8 +153,8 @@ def render_to_files(self, array_of_template_targets): def _render_with_finding_template_first(self, template_file_index): for (template_file, data_output_pairs) in template_file_index.items(): template = self.engine.get_template(template_file) - template_abs_path = utils.get_template_path( - self.template_dirs, template_file + template_abs_path = self.template_fs.geturl( + template_file, purpose="fs" ) for (data_file, output) in data_output_pairs: data = self.context.get_data(data_file) @@ -170,8 +171,8 @@ def _render_with_finding_data_first(self, data_file_index): data = self.context.get_data(data_file) for (template_file, output) in template_output_pairs: template = self.engine.get_template(template_file) - template_abs_path = utils.get_template_path( - self.template_dirs, template_file + template_abs_path = self.template_fs.geturl( + template_file, purpose="fs" ) flag = self.apply_template( template_abs_path, template, data, output From 02823a5c91aaeb6ca486b7868e2f9295eed5109c Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 21:35:14 +0100 Subject: [PATCH 138/167] :green_heart: make python 2 pass as multi_fs hates str --- moban/plugins/template.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index bb2eede5..2d66c6dc 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -85,10 +85,9 @@ def number_of_templated_files(self): def render_to_file(self, template_file, data_file, output_file): self.file_count = 1 + template_file = file_system.to_unicode(template_file) data = self.context.get_data(data_file) - template = self.engine.get_template( - file_system.to_unicode(template_file) - ) + template = self.engine.get_template(template_file) template_abs_path = self.template_fs.geturl( template_file, purpose="fs" ) @@ -154,7 +153,7 @@ def _render_with_finding_template_first(self, template_file_index): for (template_file, data_output_pairs) in template_file_index.items(): template = self.engine.get_template(template_file) template_abs_path = self.template_fs.geturl( - template_file, purpose="fs" + file_system.to_unicode(template_file), purpose="fs" ) for (data_file, output) in data_output_pairs: data = self.context.get_data(data_file) @@ -172,7 +171,7 @@ def _render_with_finding_data_first(self, data_file_index): for (template_file, output) in template_output_pairs: template = self.engine.get_template(template_file) template_abs_path = self.template_fs.geturl( - template_file, purpose="fs" + file_system.to_unicode(template_file), purpose="fs" ) flag = self.apply_template( template_abs_path, template, data, output From ff45d40d1e7443ca6f9627ab9a40575201cb9677 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 21:50:51 +0100 Subject: [PATCH 139/167] :fire: replace file searching among multiple dirs using multi-fs --- moban/utils.py | 15 --------------- tests/test_utils.py | 27 --------------------------- 2 files changed, 42 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 886f78b3..28563e94 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -91,21 +91,6 @@ def pip_install(packages): ) -def get_template_path(template_dirs, template): - for a_dir in template_dirs: - try: - template_under_dir = file_system.url_join(a_dir, template) - template_file_exists = file_system.exists( - template_under_dir - ) and file_system.is_file(template_under_dir) - - if template_file_exists: - return file_system.fs_url(template_under_dir) - except fs.errors.CreateFailed: - continue - raise exceptions.FileNotFound - - def verify_the_existence_of_directories(dirs): LOG.debug("Verifying the existence: %s", dirs) if not isinstance(dirs, list): diff --git a/tests/test_utils.py b/tests/test_utils.py index c3704826..434d3902 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,7 +11,6 @@ mkdir_p, write_file_out, file_permissions, - get_template_path, file_permissions_copy, ) from moban.exceptions import FileNotFound @@ -94,32 +93,6 @@ def test_mkdir_p(): rmtree(test_path) -@raises(FileNotFound) -def test_get_template_path_with_error(): - temp_dirs = [ - fs.path.join("tests", "fixtures", "template-tests"), - fs.path.join("tests", "abc"), - ] - template = "I-do-not-exist.jj2" - get_template_path(temp_dirs, template) - - -def test_get_template_path(): - temp_dirs = [ - fs.path.join("tests", "fixtures", "template-tests"), - fs.path.join("tests", "abc"), - fs.path.join("tests", "abc"), - ] - template = "a.jj2" - template_path = get_template_path(temp_dirs, template) - expected = "osfs://" + os.path.normcase( - os.path.abspath( - os.path.join("tests", "fixtures", "template-tests", "a.jj2") - ) - ) - eq_(template_path.lower(), expected.replace("\\", "/").lower()) - - @patch("subprocess.check_call") def test_pip_install(fake_check_all): import sys From 860dedf9b7d2dbbaadd1a12546ad493c36cfd083 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 21:57:27 +0100 Subject: [PATCH 140/167] :fire: remove unused imports. flake8 warns --- moban/utils.py | 1 - tests/test_utils.py | 1 - 2 files changed, 2 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 28563e94..333b1701 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -3,7 +3,6 @@ import errno import logging -import fs from moban import constants, exceptions, file_system LOG = logging.getLogger(__name__) diff --git a/tests/test_utils.py b/tests/test_utils.py index 434d3902..6b81c8a7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,7 +3,6 @@ import stat from shutil import rmtree -import fs.path from mock import patch from nose import SkipTest from nose.tools import eq_, raises From c867b7329152a9ace1f57e4afb94aba39dab2ce4 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 22:21:47 +0100 Subject: [PATCH 141/167] :tractor: relocate search_file function --- moban/data_loaders/manager.py | 18 +++++++++++++++++- moban/utils.py | 16 ---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index 2ae7fa68..45c518fb 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -25,7 +25,7 @@ def get_data(self, file_name): def load_data(base_dir, file_name): - abs_file_path = utils.search_file(base_dir, file_name) + abs_file_path = search_file(base_dir, file_name) data = LOADER.get_data(abs_file_path) if data is not None: parent_data = None @@ -39,3 +39,19 @@ def load_data(base_dir, file_name): return data else: return None + + +def search_file(base_dir, file_name): + the_file = file_name + if not file_system.exists(the_file): + if base_dir: + file_under_base_dir = file_system.url_join(base_dir, the_file) + if file_system.exists(file_under_base_dir): + the_file = file_system.fs_url(file_under_base_dir) + else: + raise IOError( + constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) + ) + else: + raise IOError(constants.ERROR_DATA_FILE_ABSENT % the_file) + return the_file diff --git a/moban/utils.py b/moban/utils.py index 333b1701..b5016d27 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -29,22 +29,6 @@ def merge(left, right): return left -def search_file(base_dir, file_name): - the_file = file_name - if not file_system.exists(the_file): - if base_dir: - file_under_base_dir = file_system.url_join(base_dir, the_file) - if file_system.exists(file_under_base_dir): - the_file = file_system.fs_url(file_under_base_dir) - else: - raise IOError( - constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) - ) - else: - raise IOError(constants.ERROR_DATA_FILE_ABSENT % the_file) - return the_file - - def file_permissions_copy(source, dest): source_permissions = file_permissions(source) dest_permissions = file_permissions(dest) From b31c32294fc94212c87460e11aa03889b3da5fa8 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 22:27:44 +0100 Subject: [PATCH 142/167] :tractor: code coherence, moving code to where it has stronger connections with --- moban/buffered_writer.py | 18 +++++++++++++++++- moban/utils.py | 13 ------------- tests/test_utils.py | 16 ---------------- 3 files changed, 17 insertions(+), 30 deletions(-) diff --git a/moban/buffered_writer.py b/moban/buffered_writer.py index 44b4e23f..8c57f409 100644 --- a/moban/buffered_writer.py +++ b/moban/buffered_writer.py @@ -1,6 +1,9 @@ +import sys +import os import fs import fs.path from moban import utils, file_system +PY2 = sys.version_info[0] == 2 class BufferedWriter(object): @@ -11,7 +14,7 @@ def write_file_out(self, filename, content): if file_system.is_zip_alike_url(filename): self.write_file_out_to_zip(filename, content) else: - utils.write_file_out(filename, content) + write_file_out(filename, content) def write_file_out_to_zip(self, filename, content): zip_file, file_name = file_system.url_split(filename) @@ -29,3 +32,16 @@ def write_file_out_to_zip(self, filename, content): def close(self): for fsx in self.fs_list.values(): fsx.close() + + +def write_file_out(filename, content): + if PY2 and content.__class__.__name__ == "unicode": + content = content.encode("utf-8") + + if not file_system.is_zip_alike_url(filename): + # fix me + dest_folder = os.path.dirname(filename) + if dest_folder: + utils.mkdir_p(dest_folder) + + file_system.write_bytes(filename, content) diff --git a/moban/utils.py b/moban/utils.py index b5016d27..1ad7e0c6 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -43,19 +43,6 @@ def file_permissions(afile): return file_system.file_permissions(afile) -def write_file_out(filename, content): - if PY2 and content.__class__.__name__ == "unicode": - content = content.encode("utf-8") - - if not file_system.is_zip_alike_url(filename): - # fix me - dest_folder = os.path.dirname(filename) - if dest_folder: - mkdir_p(dest_folder) - - file_system.write_bytes(filename, content) - - def mkdir_p(path): try: os.makedirs(path) diff --git a/tests/test_utils.py b/tests/test_utils.py index 6b81c8a7..30a71340 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,7 +8,6 @@ from nose.tools import eq_, raises from moban.utils import ( mkdir_p, - write_file_out, file_permissions, file_permissions_copy, ) @@ -70,21 +69,6 @@ def test_file_permission_copy_symlink(): os.unlink(test_symlink) -def test_write_file_out(): - content = b""" - helloworld - - - - - """ - test_file = "testout" - write_file_out(test_file, content) - with open(test_file, "r") as f: - content = f.read() - eq_(content, "\n helloworld\n\n\n\n\n ") - - def test_mkdir_p(): test_path = "a/b/c/d" mkdir_p(test_path) From c4e9cde18acd0fd1a72329321ff1c4987b198f20 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 22:35:09 +0100 Subject: [PATCH 143/167] :shirt: coding style update --- moban/buffered_writer.py | 4 +++- tests/test_utils.py | 6 +----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/moban/buffered_writer.py b/moban/buffered_writer.py index 8c57f409..49ca86d0 100644 --- a/moban/buffered_writer.py +++ b/moban/buffered_writer.py @@ -1,8 +1,10 @@ -import sys import os +import sys + import fs import fs.path from moban import utils, file_system + PY2 = sys.version_info[0] == 2 diff --git a/tests/test_utils.py b/tests/test_utils.py index 30a71340..0165772f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -6,11 +6,7 @@ from mock import patch from nose import SkipTest from nose.tools import eq_, raises -from moban.utils import ( - mkdir_p, - file_permissions, - file_permissions_copy, -) +from moban.utils import mkdir_p, file_permissions, file_permissions_copy from moban.exceptions import FileNotFound From 123b0ab60083902c9ceadef8f64b0a8bbeb01384 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 23:10:02 +0100 Subject: [PATCH 144/167] :fire: replace mult directory search with multi_fs whic exists already --- moban/copy/__init__.py | 8 +++----- moban/mobanfile/__init__.py | 7 ++++--- moban/mobanfile/templates.py | 21 +++++++++++---------- moban/utils.py | 10 ---------- 4 files changed, 18 insertions(+), 28 deletions(-) diff --git a/moban/copy/__init__.py b/moban/copy/__init__.py index 4a9e2deb..0754c751 100644 --- a/moban/copy/__init__.py +++ b/moban/copy/__init__.py @@ -1,4 +1,4 @@ -from moban import utils, constants, file_system +from moban import constants, file_system from lml.plugin import PluginInfo @@ -23,10 +23,8 @@ def __init__(self, template_dirs, extensions=None): self.template_dirs = template_dirs def get_template(self, template_file): - real_file_name = utils.find_file_in_template_dirs( - template_file, self.template_dirs - ) - return file_system.read_binary(real_file_name) + multi_fs = file_system.get_multi_fs(self.template_dirs) + return multi_fs.readbytes(file_system.to_unicode(template_file)) def get_template_from_string(self, string): return string diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index b96d439d..bf42ec50 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -6,7 +6,7 @@ from moban import plugins, reporter, constants, file_system from lml.utils import do_import from moban.repo import git_clone -from moban.utils import merge, pip_install +from moban.utils import merge, pip_install, verify_the_existence_of_directories from moban.deprecated import deprecated from moban.definitions import GitRequire from moban.plugins.template import expand_template_directories @@ -62,8 +62,9 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): # call expand template directory always after handle require please # the penalty is: newly clone repos are not visible - merged_options[constants.LABEL_TMPL_DIRS] = list( - expand_template_directories(merged_options[constants.LABEL_TMPL_DIRS]) + # one more note: verify_the_existence_of_directories will remove non-exist dirs + merged_options[constants.LABEL_TMPL_DIRS] = verify_the_existence_of_directories( + list(expand_template_directories(merged_options[constants.LABEL_TMPL_DIRS])) ) extensions = moban_file_configurations.get(constants.LABEL_EXTENSIONS) if extensions: diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index b12b24a5..32e7b2b0 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -1,18 +1,19 @@ import logging from moban import reporter, file_system -from moban.utils import find_file_in_template_dirs log = logging.getLogger(__name__) def handle_template(template_file, output, template_dirs): log.info("handling %s" % template_file) - + multi_fs = file_system.get_multi_fs(template_dirs) if template_file.endswith("**"): source_dir = template_file[:-3] - src_path = find_file_in_template_dirs(source_dir, template_dirs) - if src_path: + if multi_fs.exists(source_dir): + src_path = multi_fs.geturl( + file_system.to_unicode(source_dir), purpose="fs" + ) for a_triple in _listing_directory_files_recusively( source_dir, src_path, output ): @@ -22,15 +23,15 @@ def handle_template(template_file, output, template_dirs): "{0} cannot be found".format(template_file) ) else: - template_file_on_disk = find_file_in_template_dirs( - template_file, template_dirs - ) - - if template_file_on_disk is None: + template_file = file_system.to_unicode(template_file) + if not multi_fs.exists(template_file): reporter.report_error_message( "{0} cannot be found".format(template_file) ) - elif file_system.is_dir(template_file_on_disk): + elif multi_fs.isdir(template_file): + template_file_on_disk = multi_fs.geturl( + file_system.to_unicode(template_file), purpose="fs" + ) for a_triple in _list_dir_files( template_file, template_file_on_disk, output ): diff --git a/moban/utils.py b/moban/utils.py index 1ad7e0c6..0a73f6ee 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -85,13 +85,3 @@ def verify_the_existence_of_directories(dirs): constants.MESSAGE_DIR_NOT_EXIST % directory ) return results - - -def find_file_in_template_dirs(src, template_dirs): - LOG.debug(template_dirs) - for folder in template_dirs: - path = file_system.url_join(folder, src) - if file_system.exists(path): - return path - else: - return None From fcb6d6ede437aefd5a5139e2bc28b775e46d7a6c Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 23:10:56 +0100 Subject: [PATCH 145/167] :shirt: coding style update --- moban/mobanfile/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index bf42ec50..cae13c74 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -63,8 +63,14 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): # call expand template directory always after handle require please # the penalty is: newly clone repos are not visible # one more note: verify_the_existence_of_directories will remove non-exist dirs - merged_options[constants.LABEL_TMPL_DIRS] = verify_the_existence_of_directories( - list(expand_template_directories(merged_options[constants.LABEL_TMPL_DIRS])) + merged_options[ + constants.LABEL_TMPL_DIRS + ] = verify_the_existence_of_directories( + list( + expand_template_directories( + merged_options[constants.LABEL_TMPL_DIRS] + ) + ) ) extensions = moban_file_configurations.get(constants.LABEL_EXTENSIONS) if extensions: From 90738634f80950a5a2fa6d15b29dbbc06f999b52 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 23:19:37 +0100 Subject: [PATCH 146/167] :tractor: function coherence, file permissions belong to file system. utils is a bit distant --- moban/file_system.py | 11 +++++++ moban/hashstore.py | 4 +-- moban/plugins/template.py | 4 ++- moban/utils.py | 14 --------- tests/test_file_system.py | 61 +++++++++++++++++++++++++++++++++++++- tests/test_utils.py | 62 +-------------------------------------- 6 files changed, 77 insertions(+), 79 deletions(-) diff --git a/moban/file_system.py b/moban/file_system.py index 42e8b921..9f8a9001 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -6,6 +6,7 @@ import fs import fs.path +from moban import exceptions from fs.multifs import MultiFS try: @@ -214,7 +215,17 @@ def is_zip_alike_url(url): return False +def file_permissions_copy(source, dest): + source_permissions = file_permissions(source) + dest_permissions = file_permissions(dest) + + if source_permissions != dest_permissions: + os.chmod(dest, source_permissions) + + def file_permissions(url): + if not exists(url): + raise exceptions.FileNotFound(url) if sys.platform == "win32": return 0 elif is_zip_alike_url(url): diff --git a/moban/hashstore.py b/moban/hashstore.py index 6adbd604..6cd5c36c 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -33,7 +33,7 @@ def is_file_changed(self, file_name, file_content, source_template): def _is_source_updated(self, file_name, file_content, source_template): changed = True content = _mix( - file_content, oct(utils.file_permissions(source_template)) + file_content, oct(file_system.file_permissions(source_template)) ) content_hash = get_hash(content) if os.path.exists(file_name): @@ -57,7 +57,7 @@ def save_hashes(self): def get_file_hash(afile): content = file_system.read_bytes(afile) - content = _mix(content, oct(utils.file_permissions(afile))) + content = _mix(content, oct(file_system.file_permissions(afile))) return get_hash(content) diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 2d66c6dc..ab4972b7 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -131,7 +131,9 @@ def apply_template(self, template_abs_path, template, data, output_file): output_file, rendered_content ) if not file_system.is_zip_alike_url(output_file): - utils.file_permissions_copy(template_abs_path, output_file) + file_system.file_permissions_copy( + template_abs_path, output_file + ) return flag except exceptions.FileNotFound as e: log.exception(e) diff --git a/moban/utils.py b/moban/utils.py index 0a73f6ee..e7d36608 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -29,20 +29,6 @@ def merge(left, right): return left -def file_permissions_copy(source, dest): - source_permissions = file_permissions(source) - dest_permissions = file_permissions(dest) - - if source_permissions != dest_permissions: - os.chmod(dest, source_permissions) - - -def file_permissions(afile): - if not file_system.exists(afile): - raise exceptions.FileNotFound(afile) - return file_system.file_permissions(afile) - - def mkdir_p(path): try: os.makedirs(path) diff --git a/tests/test_file_system.py b/tests/test_file_system.py index 445710f8..40a81be5 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -1,7 +1,11 @@ import os +import sys +import stat from moban import file_system -from nose.tools import eq_ +from nose import SkipTest +from nose.tools import eq_, raises +from moban.exceptions import FileNotFound LOCAL_FOLDER = "tests/fixtures" LOCAL_FILE = LOCAL_FOLDER + "/a.jj2" @@ -184,3 +188,58 @@ def test_url_join(): for parent, child, expected_path in URL_JOIN_TEST_FIXTURES: actual = file_system.url_join(parent, child) eq_(actual, expected_path) + + +def create_file(test_file, permission): + with open(test_file, "w") as f: + f.write("test") + + os.chmod(test_file, permission) + + +def test_file_permission_copy(): + if sys.platform == "win32": + raise SkipTest("No actual chmod on windows") + test_source = "test_file_permission_copy1" + test_dest = "test_file_permission_copy2" + create_file(test_source, 0o755) + create_file(test_dest, 0o646) + file_system.file_permissions_copy(test_source, test_dest) + eq_( + stat.S_IMODE(os.lstat(test_source).st_mode), + stat.S_IMODE(os.lstat(test_dest).st_mode), + ) + os.unlink(test_source) + os.unlink(test_dest) + + +def file_permissions_disabled_on_windows(): + if sys.platform == "win32": + permissions = file_system.file_permissions("abc") + eq_("no-permission-support", permissions) + else: + raise SkipTest("No test required") + + +@raises(FileNotFound) +def test_file_permissions_file_not_found(): + file_system.file_permissions("I does not exist") + + +def test_file_permission_copy_symlink(): + if sys.platform == "win32": + raise SkipTest("No symlink on windows") + test_source = "test_file_permission_copy1" + test_dest = "test_file_permission_copy2" + test_symlink = "test_file_permission_symlink" + create_file(test_source, 0o046) + os.symlink(test_source, test_symlink) + create_file(test_dest, 0o646) + file_system.file_permissions_copy(test_source, test_dest) + eq_( + stat.S_IMODE(os.lstat(test_source).st_mode), + stat.S_IMODE(os.lstat(test_dest).st_mode), + ) + os.unlink(test_source) + os.unlink(test_dest) + os.unlink(test_symlink) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0165772f..156fcfc6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,68 +1,8 @@ import os -import sys -import stat from shutil import rmtree from mock import patch -from nose import SkipTest -from nose.tools import eq_, raises -from moban.utils import mkdir_p, file_permissions, file_permissions_copy -from moban.exceptions import FileNotFound - - -def create_file(test_file, permission): - with open(test_file, "w") as f: - f.write("test") - - os.chmod(test_file, permission) - - -def test_file_permission_copy(): - if sys.platform == "win32": - raise SkipTest("No actual chmod on windows") - test_source = "test_file_permission_copy1" - test_dest = "test_file_permission_copy2" - create_file(test_source, 0o755) - create_file(test_dest, 0o646) - file_permissions_copy(test_source, test_dest) - eq_( - stat.S_IMODE(os.lstat(test_source).st_mode), - stat.S_IMODE(os.lstat(test_dest).st_mode), - ) - os.unlink(test_source) - os.unlink(test_dest) - - -def file_permissions_disabled_on_windows(): - if sys.platform == "win32": - permissions = file_permissions("abc") - eq_("no-permission-support", permissions) - else: - raise SkipTest("No test required") - - -@raises(FileNotFound) -def test_file_permissions_file_not_found(): - file_permissions("I does not exist") - - -def test_file_permission_copy_symlink(): - if sys.platform == "win32": - raise SkipTest("No symlink on windows") - test_source = "test_file_permission_copy1" - test_dest = "test_file_permission_copy2" - test_symlink = "test_file_permission_symlink" - create_file(test_source, 0o046) - os.symlink(test_source, test_symlink) - create_file(test_dest, 0o646) - file_permissions_copy(test_source, test_dest) - eq_( - stat.S_IMODE(os.lstat(test_source).st_mode), - stat.S_IMODE(os.lstat(test_dest).st_mode), - ) - os.unlink(test_source) - os.unlink(test_dest) - os.unlink(test_symlink) +from moban.utils import mkdir_p def test_mkdir_p(): From e3d7d74e902d2fd476b702de9fa28f97e53218ca Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 23:19:57 +0100 Subject: [PATCH 147/167] :shirt: coding style --- tests/test_file_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_file_system.py b/tests/test_file_system.py index 40a81be5..5b2eb26b 100644 --- a/tests/test_file_system.py +++ b/tests/test_file_system.py @@ -2,8 +2,8 @@ import sys import stat -from moban import file_system from nose import SkipTest +from moban import file_system from nose.tools import eq_, raises from moban.exceptions import FileNotFound From 1316c3bfbceeac815e1fcb3d09ebe2b1449a9ee0 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 23:22:17 +0100 Subject: [PATCH 148/167] :green_heart: fix python2 again. non-unicode string --- moban/mobanfile/templates.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 32e7b2b0..f68baa59 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -7,12 +7,13 @@ def handle_template(template_file, output, template_dirs): log.info("handling %s" % template_file) + template_file = file_system.to_unicode(template_file) multi_fs = file_system.get_multi_fs(template_dirs) if template_file.endswith("**"): source_dir = template_file[:-3] if multi_fs.exists(source_dir): src_path = multi_fs.geturl( - file_system.to_unicode(source_dir), purpose="fs" + source_dir, purpose="fs" ) for a_triple in _listing_directory_files_recusively( source_dir, src_path, output @@ -23,14 +24,13 @@ def handle_template(template_file, output, template_dirs): "{0} cannot be found".format(template_file) ) else: - template_file = file_system.to_unicode(template_file) if not multi_fs.exists(template_file): reporter.report_error_message( "{0} cannot be found".format(template_file) ) elif multi_fs.isdir(template_file): template_file_on_disk = multi_fs.geturl( - file_system.to_unicode(template_file), purpose="fs" + template_file, purpose="fs" ) for a_triple in _list_dir_files( template_file, template_file_on_disk, output From c556482562cf2fb9fc3b217f6cc00355043f3d08 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 23:25:36 +0100 Subject: [PATCH 149/167] :fire: remove unused import --- moban/hashstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/hashstore.py b/moban/hashstore.py index 6cd5c36c..1fef3d6e 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -3,7 +3,7 @@ import json import hashlib -from moban import utils, constants, file_system +from moban import constants, file_system PY2 = sys.version_info[0] == 2 From 411283f4b5d499694c71ccf04bc8760a1968e3d2 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Aug 2019 23:32:39 +0100 Subject: [PATCH 150/167] :shirt: coding style update using black :hammer: replace os.path in moban files :fire: remove one more os line --- moban/buffered_writer.py | 1 - moban/definitions.py | 6 +++--- moban/hashstore.py | 14 +++++++++----- moban/mobanfile/templates.py | 4 +--- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/moban/buffered_writer.py b/moban/buffered_writer.py index 49ca86d0..ff980895 100644 --- a/moban/buffered_writer.py +++ b/moban/buffered_writer.py @@ -41,7 +41,6 @@ def write_file_out(filename, content): content = content.encode("utf-8") if not file_system.is_zip_alike_url(filename): - # fix me dest_folder = os.path.dirname(filename) if dest_folder: utils.mkdir_p(dest_folder) diff --git a/moban/definitions.py b/moban/definitions.py index 8ae4ea45..a78a8f07 100644 --- a/moban/definitions.py +++ b/moban/definitions.py @@ -1,5 +1,3 @@ -import os - from moban import constants @@ -54,7 +52,9 @@ def __init__( def set_template_type(self, new_template_type): self.template_type = new_template_type if self.original_output.endswith(self.template_type): - self.output, _ = os.path.splitext(self.original_output) + self.output = self.original_output.replace( + "." + self.template_type, "" + ) else: self.output = self.original_output diff --git a/moban/hashstore.py b/moban/hashstore.py index 1fef3d6e..636e7165 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -1,4 +1,3 @@ -import os import sys import json import hashlib @@ -12,9 +11,14 @@ class HashStore: IGNORE_CACHE_FILE = False def __init__(self): - self.cache_file = constants.DEFAULT_MOBAN_CACHE_FILE - if os.path.exists(self.cache_file) and self.IGNORE_CACHE_FILE is False: - with open(self.cache_file, "r") as f: + self.cache_file = file_system.to_unicode( + constants.DEFAULT_MOBAN_CACHE_FILE + ) + if ( + file_system.exists(self.cache_file) + and self.IGNORE_CACHE_FILE is False + ): + with file_system.open_file(self.cache_file) as f: self.hashes = json.load(f) else: self.hashes = {} @@ -36,7 +40,7 @@ def _is_source_updated(self, file_name, file_content, source_template): file_content, oct(file_system.file_permissions(source_template)) ) content_hash = get_hash(content) - if os.path.exists(file_name): + if file_system.exists(file_name): if file_name in self.hashes: if content_hash == self.hashes[file_name]: changed = False diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index f68baa59..80e3f6c5 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -12,9 +12,7 @@ def handle_template(template_file, output, template_dirs): if template_file.endswith("**"): source_dir = template_file[:-3] if multi_fs.exists(source_dir): - src_path = multi_fs.geturl( - source_dir, purpose="fs" - ) + src_path = multi_fs.geturl(source_dir, purpose="fs") for a_triple in _listing_directory_files_recusively( source_dir, src_path, output ): From f5f62069cdd914062831f000c0162d883e53c59b Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 11 Aug 2019 14:23:38 +0100 Subject: [PATCH 151/167] :sparkles: update api interface using fs.multifs.MultiFS instance --- moban/copy/__init__.py | 6 +++--- moban/jinja2/engine.py | 8 +++----- moban/plugins/template.py | 10 +++++----- tests/test_copy_encoding.py | 4 +++- tests/test_copy_engine.py | 4 +++- tests/test_jinja2_engine.py | 7 +++++-- tests/test_jinja2_extensions.py | 4 +++- 7 files changed, 25 insertions(+), 18 deletions(-) diff --git a/moban/copy/__init__.py b/moban/copy/__init__.py index 0754c751..08cd1cf4 100644 --- a/moban/copy/__init__.py +++ b/moban/copy/__init__.py @@ -19,11 +19,11 @@ class ContentForwardEngine(object): templating mechanism. """ - def __init__(self, template_dirs, extensions=None): - self.template_dirs = template_dirs + def __init__(self, template_fs, extensions=None): + self.template_fs = template_fs def get_template(self, template_file): - multi_fs = file_system.get_multi_fs(self.template_dirs) + multi_fs = self.template_fs return multi_fs.readbytes(file_system.to_unicode(template_file)) def get_template_from_string(self, string): diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index 652c41ee..2c226df4 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -57,7 +57,7 @@ def __init__(self): constants.TEMPLATE_ENGINE_EXTENSION, tags=["jinja2", "jinja", "jj2", "j2"] ) class Engine(object): - def __init__(self, template_dirs, options=None): + def __init__(self, template_fs, options=None): """ Contruct a jinja2 template engine @@ -66,11 +66,9 @@ def __init__(self, template_dirs, options=None): :param list temp_dirs: a list of template directories :param dict options: a dictionary containing environmental parameters """ - LOG.debug("Jinja template dirs: %s", template_dirs) + LOG.debug("Jinja template engine started") load_jinja2_extensions() - self.template_dirs = template_dirs - filesystem = file_system.get_multi_fs(template_dirs) - template_loader = FSLoader(filesystem) + template_loader = FSLoader(template_fs) env_params = dict( loader=template_loader, keep_trailing_newline=True, diff --git a/moban/plugins/template.py b/moban/plugins/template.py index ab4972b7..6649231b 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -44,8 +44,9 @@ def get_engine(self, template_type, template_dirs, context_dirs): engine_cls = self.load_me_now(template_type) engine_extensions = self.extensions.get(template_type) options = dict(extensions=engine_extensions) - engine = engine_cls(template_dirs, options) - return MobanEngine(template_dirs, context_dirs, engine) + template_fs = file_system.get_multi_fs(template_dirs) + engine = engine_cls(template_fs, options) + return MobanEngine(template_fs, context_dirs, engine) def get_primary_key(self, template_type): for key, item in self.options_registry.items(): @@ -62,11 +63,10 @@ def raise_exception(self, key): class MobanEngine(object): - def __init__(self, template_dirs, context_dirs, engine): + def __init__(self, template_fs, context_dirs, engine): context_dirs = expand_template_directory(context_dirs) self.context = Context(context_dirs) - self.template_fs = file_system.get_multi_fs(template_dirs) - self.template_dirs = template_dirs + self.template_fs = template_fs self.engine = engine self.templated_count = 0 self.file_count = 0 diff --git a/tests/test_copy_encoding.py b/tests/test_copy_encoding.py index bcad3674..b388f8c7 100644 --- a/tests/test_copy_encoding.py +++ b/tests/test_copy_encoding.py @@ -1,4 +1,5 @@ import fs.path +from moban import file_system from moban.copy import ContentForwardEngine from nose.tools import eq_ @@ -6,7 +7,8 @@ class TestCopyEncoding: def setUp(self): template_path = fs.path.join("tests", "fixtures") - self.engine = ContentForwardEngine([template_path]) + template_fs = file_system.get_multi_fs([template_path]) + self.engine = ContentForwardEngine(template_fs) def test_encoding_of_template(self): template_content = self.engine.get_template("coala_color.svg") diff --git a/tests/test_copy_engine.py b/tests/test_copy_engine.py index eecf9a95..fae1c5f1 100644 --- a/tests/test_copy_engine.py +++ b/tests/test_copy_engine.py @@ -1,5 +1,6 @@ import os +from moban import file_system from moban.copy import ContentForwardEngine from nose.tools import eq_ @@ -7,7 +8,8 @@ class TestContentForwardEngine: def setUp(self): template_path = os.path.join("tests", "fixtures") - self.engine = ContentForwardEngine([template_path]) + fs = file_system.get_multi_fs([template_path]) + self.engine = ContentForwardEngine(fs) def test_get_template(self): template_content = self.engine.get_template("copier-test01.csv") diff --git a/tests/test_jinja2_engine.py b/tests/test_jinja2_engine.py index 8996d951..5238621a 100644 --- a/tests/test_jinja2_engine.py +++ b/tests/test_jinja2_engine.py @@ -1,12 +1,14 @@ import os +from moban import file_system from nose.tools import eq_ from moban.jinja2.engine import Engine def test_jinja2_template(): path = os.path.join("tests", "fixtures", "jinja_tests") - engine = Engine([path]) + fs = file_system.get_multi_fs([path]) + engine = Engine(fs) template = engine.get_template("file_tests.template") data = dict(test="here") result = engine.apply_template(template, data, None) @@ -16,7 +18,8 @@ def test_jinja2_template(): def test_jinja2_template_string(): path = os.path.join("tests", "fixtures", "jinja_tests") - engine = Engine([path]) + fs = file_system.get_multi_fs([path]) + engine = Engine(fs) template = engine.get_template_from_string("{{test}}") data = dict(test="here") result = engine.apply_template(template, data, None) diff --git a/tests/test_jinja2_extensions.py b/tests/test_jinja2_extensions.py index 5ef492ba..48ddbdb9 100644 --- a/tests/test_jinja2_extensions.py +++ b/tests/test_jinja2_extensions.py @@ -1,5 +1,6 @@ import os +from moban import file_system from nose.tools import eq_ from moban.jinja2.engine import Engine from moban.plugins.template import MobanEngine @@ -11,7 +12,8 @@ def test_globals(): test_dict = dict(hello="world") jinja_global("test", test_dict) path = os.path.join("tests", "fixtures", "globals") - engine = MobanEngine([path], path, Engine([path])) + template_fs = file_system.get_multi_fs([path]) + engine = MobanEngine(template_fs, path, Engine(template_fs)) engine.render_to_file("basic.template", "basic.yml", output) with open(output, "r") as output_file: content = output_file.read() From 20282f847016400816b931b4b886f9b5a88a8eff Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 11 Aug 2019 14:27:53 +0100 Subject: [PATCH 152/167] :fire: remove unused import --- moban/jinja2/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index 2c226df4..04aee6a1 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -2,7 +2,7 @@ import logging from importlib import import_module -from moban import constants, exceptions, file_system +from moban import constants, exceptions from jinja2 import Template, Environment from lml.loader import scan_plugins_regex from lml.plugin import PluginInfo, PluginManager From a0db5d2b46815435ee42537a769654b8f5adb6e9 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 11 Aug 2019 17:05:06 +0100 Subject: [PATCH 153/167] :books: update docstring and extension guide --- docs/extension.rst | 2 +- moban/copy/__init__.py | 3 +-- moban/jinja2/engine.py | 5 +++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/extension.rst b/docs/extension.rst index aa4d6a98..b7ea1bc2 100644 --- a/docs/extension.rst +++ b/docs/extension.rst @@ -48,7 +48,7 @@ is an example starting point for any template engine. constants.TEMPLATE_ENGINE_EXTENSION, tags=["file", "extensions", "for", "your", "template"] ) class Engine(object): - def __init__(self, template_dirs): + def __init__(self, template_fs, options=None): """ A list template directories will be given to your engine class """ diff --git a/moban/copy/__init__.py b/moban/copy/__init__.py index 08cd1cf4..36c36dde 100644 --- a/moban/copy/__init__.py +++ b/moban/copy/__init__.py @@ -23,8 +23,7 @@ def __init__(self, template_fs, extensions=None): self.template_fs = template_fs def get_template(self, template_file): - multi_fs = self.template_fs - return multi_fs.readbytes(file_system.to_unicode(template_file)) + return self.template_fs.readbytes(file_system.to_unicode(template_file)) def get_template_from_string(self, string): return string diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index 04aee6a1..eea38428 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -61,9 +61,10 @@ def __init__(self, template_fs, options=None): """ Contruct a jinja2 template engine - A list template directories will be given to your engine class + an instance of fs.multifs.MultiFS will be given and straightaway it is given + to jinja2.FSLoader. - :param list temp_dirs: a list of template directories + :param fs.multifs.MultiFS template_fs: a MultiFS instance or a FS instance :param dict options: a dictionary containing environmental parameters """ LOG.debug("Jinja template engine started") From 6ea5cb822a3b45afad8f8c85e1b92768cc9c65bc Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 11 Aug 2019 17:35:31 +0100 Subject: [PATCH 154/167] :shirt: coding style update --- moban/copy/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moban/copy/__init__.py b/moban/copy/__init__.py index 36c36dde..e617366a 100644 --- a/moban/copy/__init__.py +++ b/moban/copy/__init__.py @@ -23,7 +23,9 @@ def __init__(self, template_fs, extensions=None): self.template_fs = template_fs def get_template(self, template_file): - return self.template_fs.readbytes(file_system.to_unicode(template_file)) + return self.template_fs.readbytes( + file_system.to_unicode(template_file) + ) def get_template_from_string(self, string): return string From 8815f7a035742e0f93cc1d618c6fd54003303f6c Mon Sep 17 00:00:00 2001 From: jaska Date: Fri, 16 Aug 2019 08:17:25 +0100 Subject: [PATCH 155/167] gitfs2 - the final syntax for repo (#312) * :books: update level 10 moban dependency as git repo. we will use gitfs2 from moremoban * :tractor: relocate files * :fire: remove repo file system openner :tractor: relocate old code to deprecated * :shirt: update coding style * :green_heart: fix unit test import * :tractor: relocate deprecated code to deprecated folder and will start remove later * :shirt: pass moban_command check * :shirt: update coding style * :books: add any location to our vision * :fire: remove pypi requires from moban. pypifs takes care of it now. * :tractor: relocated the pip_install code to deprecated folder. ready for future clean up * :microscope: level 21 can read from tar too * :microscope: test copy from a tar * :shirt: update coding style * :fire: remove useless test fixtures * :books: update regresssion test readme :books: update level 9 and 10 docs about packages required to support * :books: update diagram command * :books: update architecture diagram --- Makefile | 7 ++ README.rst | 16 ++-- docs/conf.py | 1 + docs/engine.png | Bin 13110 -> 0 bytes docs/engine.uml | 35 +++++++- docs/extension.rst | 2 +- docs/images/engine.svg | 69 +++++++++++++++ .../.moban.yml | 6 +- .../README.rst | 33 ++++--- .../.moban.yml | 2 - .../README.rst | 29 ++---- moban/constants.py | 6 +- moban/definitions.py | 32 ------- moban/deprecated.py | 15 ---- moban/deprecated/__init__.py | 56 ++++++++++++ moban/{ => deprecated}/repo.py | 0 moban/fs_openers.py | 18 ---- moban/main.py | 1 - moban/mobanfile/__init__.py | 8 +- moban/plugins/template.py | 4 +- moban/utils.py | 8 -- rnd_requirements.txt | 1 + tests/{ => deprecated}/test_repo.py | 8 +- tests/mobanfile/test_mobanfile.py | 3 +- .../.moban.yml | 19 ++++ .../README.rst | 83 ++++++++++++++++++ .../template-sources.tar | Bin .../.moban.yml | 19 ++++ .../README.rst | 24 +++++ .../template-sources.tar | Bin 0 -> 10752 bytes tests/test_definitions.py | 3 +- tests/test_engine.py | 2 +- tests/test_regression.py | 69 +++++++++++++++ tests/test_utils.py | 2 +- 34 files changed, 443 insertions(+), 138 deletions(-) delete mode 100644 docs/engine.png create mode 100644 docs/images/engine.svg delete mode 100644 moban/deprecated.py create mode 100644 moban/deprecated/__init__.py rename moban/{ => deprecated}/repo.py (100%) delete mode 100644 moban/fs_openers.py rename tests/{ => deprecated}/test_repo.py (98%) create mode 100644 tests/regression_tests/level-21-b-copy-templates-into-a-tar/.moban.yml create mode 100644 tests/regression_tests/level-21-b-copy-templates-into-a-tar/README.rst rename {docs/level-21-copy-templates-into-an-alien-file-system => tests/regression_tests/level-21-b-copy-templates-into-a-tar}/template-sources.tar (100%) create mode 100644 tests/regression_tests/level-21-c-copy-templates-from-a-tar/.moban.yml create mode 100644 tests/regression_tests/level-21-c-copy-templates-from-a-tar/README.rst create mode 100644 tests/regression_tests/level-21-c-copy-templates-from-a-tar/template-sources.tar diff --git a/Makefile b/Makefile index b7e5f2c3..5d9ead41 100644 --- a/Makefile +++ b/Makefile @@ -27,3 +27,10 @@ format: git diff black -l 79 tests git diff + +uml: + plantuml -tsvg -o ./images/ docs/*.uml + + +doc: uml + sphinx-build -b html docs build diff --git a/README.rst b/README.rst index 528b5aec..cbb1034c 100644 --- a/README.rst +++ b/README.rst @@ -25,13 +25,15 @@ moban - 模板 Yet another jinja2 cli command for static text generation **moban** brings the high performance template engine (JINJA2) for web into -static text generation. It is used in `pyexcel` and `coala` project to keep documentation -consistent across the documentations of individual libraries. - -Our vision is: any template, any data. Our current architecture enables moban -to plugin any python template engine: mako, handlebars, velocity, haml, slim and tornado -and to plugin any data format: json and yaml. Please look at our issues. We have many -more template engines and data format on the road map. +static text generation. It is used in `pyexcel` and `coala` project to keep +documentation consistent across the documentations of individual libraries. + +Our vision is: any template, any data in any location. Our current architecture +enables moban to plugin any python template engine: mako, handlebars, velocity, +haml, slim and tornado, to plugin any data format: json and yaml, and in +any location: zip, git, pypi package, s3, etc. Please +look at our issues. We have many more template engines and data format on the +road map. Installation ================================================================================ diff --git a/docs/conf.py b/docs/conf.py index 8f4f4e27..24bc782d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -69,6 +69,7 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} + # TODO: html_theme not configurable upstream html_theme = 'default' diff --git a/docs/engine.png b/docs/engine.png deleted file mode 100644 index 9cc99d435e54a188c6f92db3dc2aeda062c75e2e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13110 zcmZ8|1z1#F*ER-7BPAdLBGTOrD&08@!T{3U-Abp30@5g*L&HdS4BZU^(kZ3%zj>bb z{r~U#=emH4v(GuZ)?RDf_qr!sRapiH^C>0@3JQ*#tfV>$$~__QBZqz;TnS2WGXZ}X zU8J;KjO`sfZA{HvP-IN)Oq~o}Oii8}c|Nyrad8l0XLqnMv~zK_wP7>1x5eiWdC$*sbEEZ2^w&>ADiRki|BvTS5dz zSNm>?nr1$hb_b#xCsQ>UZ|!W->3G#?$SF%aSyWX{1B!$et$)eoYcqWyjw$A=H=pwS zvK?Kkh6`KGDifo%Ji6jev&QRV1p3!r}_b0!t& zJR}i386F*GY8HI1buRYt=grtvh|uV`tU>HoW)BEaV7D5t`Cbzm z>{|6sboc!bM(1gKI%>*rT|Y%ucjjgVct^xWxGPbtz7R>fjn`^gEGN`)zv=4_8Je%N z<;fK74zI-SojA3>*WM#bLP6n_l9PO`=`p*Ph6Pi4dwct3o(?+~=A6|1EvZ)p+bL2c zR@VagGCI9Ks280(hGeTOr*YJe~o2B}T^p`Xd>$iqOl^gwA zsbgC%Y~C5i*A0_JQ+64RY~H?8-X#Y*m$`rF!zEY3Px3KkU$(_dzQBjSBv_Vz5sMBb z2$w{BmJMPKmwW+}`{&|aTet&l>^~Pl+dmH!piITZ5!t6RFf&K)%8y#YK7625;W6&Q zu3!##@TFmkDKhTUlQXd{#sq$G?-8$3)JQ8+Rurl0Np{o2~x*nShHs z+yN3$Tv8Hi1B0bj5ESR4^!{u$^D!a9 zYNC+%>?J+@RBAmJRB3t5=(73t`XB^{{FPWw$KYU^aO8U5-+qU}B<~XKgJiYWukT&p z?k-iBV%^OIq7Q3tYb%QmXC}wT}@NO5E@2 zcu3sXzf`}mg_W=oSVI4RiqlHOK1yI^Q_~GxufB&0Sm@Ep#n09UhrNn)X$6!TKM6^J%-NGe)$8iUTfXN6;)LoD$W~&wEq~)B!qN^ zqe;%Y3g`0p__*_l2WLMQiEH8nja$94>Hb(A-rWMl@6JUr2JRw$ z6;t(8_N96wgo*_0R#LquI8$0jN|cn8417qHYazKxbtTK#9)dHe_XxyHf4!L+or zrlzK@jmocq>#^#b7#L7uAOh#${%oCoUOqcKK1YQr+eBT}HQz@azTG96t6AMh8kVb+ zk`Nacm!6JmwBwbQ*@Vb6!S{N=m@gzE@}!*%ps=Hi`l_!Xgm7XOH=nuna%Pq`hzS*o1E_S=?%k4xxg{ zuGUjo^Ur8`-WQ2r5sXVPFI-V8Oc}Lh9TEh$Su2>x)6OawBjj~Iy<49#G0sW3eg9QO zRrOPw0ug30Yb;~D9eeaL(?Epc+ukmDe*f%0(uoF!+{_$rX*HM@^n;j}4fm^0$UMZWZgqz)|GYu)l+hQrl|x5e2VXf>?F z;W}=%*Xsj`rhW5bmo_dgCv1vn3|fpU5&xQ?1&COcRXJ26bjq<;e4r8lB_|m8zvq5yQT3DZSBr9I`|2>+imO!8;mQRp14TG!S#e6qOq0NP$JA@ zYAuR&G(IITk`I@xK%?MhW>oL?BSaIMX6J6nyZ`K2UyP|JNPP)tzZ#LbwVEvMZN8b< z8m4Z3|6b}a^MlaJV1{_KpxNH6r>M7^?ab)#XobC(SN^nRNbq@bvRk0;=DOL&zF6?} z>BZ^h?PW7%3{}Q57QD69S~}`)wsOtm=oclQrP^a@QiiAfZm(Y&%zoBdt~2WF*Ke$& zrQ45+f-PkRe}DCJXzW9fFd7I&Ht#z(g zR)hQ|+0R)u_`dP7=$ueJ4h!GOQ81hi-2H?rbQ;gRkiugL(@H=-J8#FUBLALrjj=<>mv0wJs~%}k#>F|(+aHM~4q zizeqcI=|W%{>o{LX5Bgu31OAYk}WbRXQUlnR?{7&gfTPuwK4=DjczE!{ieqY_-s(C zEsxn&j~jfF7V~3U;d)EPm*>X@i=KH~0jPsEFX-s}|F{`JSU6G1k|G4#2U40foW*A! zEA#R3eUihJjd^NH-A?BBd*wxiF;>+6GwJIKYnhr)n~9N;gh~GQn35SQ%#cxT&+yCj zg&pE8uq+jnGUHjEx*goybKy*T9y9wn;b#OJ4*umSH-(@R%6%s5;ctrsQ){YkH>b-S z_ht*IL(PegcxO|<> zDz2Qwg^A0(dfNDBvHqLhU9XYJh{<9TzIR9z`nCBVz75t2IsrSS&C-q0e~EK_6vJYc z*y;n0j#XaBEB41=Ld&BRbCxeITyrar1r(J=PFLPdb}6d&zhl6Gem@+jT~tY(Zo9EE z=O|()Zj+;3`Q7@=rII2UzrpaaCV_2TvzFw8xbDz+8?n@Ll6Dk)<-LZ@6k`Y7#RO z*e&hqD$#?{$+i=3t)dVOwR~HSXO684EUJ>1Zjr#{MtjluZ!?#S5Zzmo=rCF@8?b}}vjz$Q~w6xIf zTB(lZMx);MSCk)_GGgf*6!mW4%wuJ`*m;u5-kdt0B}3xj_{tresW87ApHETNzmGRm z+BvT;{j+&%<$3~(qU7CNp5f@Giq`Y9vL<2-rRV(4XSKwGZ)QAUOI^&CGj_?aSjqpY z!;G$XaI|KZ55qR9wWE+RwOwjzP=>IuYLGcYp9ihJrmJvs&j`?tntMRgdfF))Hk=tu z?R>sGk`>n)c#C(<<_tm8k_>DojT|KJoUXOc+0`|F=QN&I#Z0Cn;tn=|%I|N}bgfCE zT`aq;+??H){Nnys8-*<;Or*Kmc!luq@T35tmR=j}-wuDdc#C}`zL1vUBQWw3O9$ogkrZ>BWE za;Yizh@&No!?R*yBO_+&kWkQ-lUABkk!7T>Jp{n`*?-G2*LzqbvJ;kXNnfE%0VIV5!<3KSBE5EKg66?VQ)|!Q_ z)^b1PSmSw|e?z3=;75l_vgviN5y+!o;^jYueoW*t8s5OqGXLE#bTp%IjEG=jWo4zK zqhn!V37f;@ufSbI(1wMo@Bh&gv_L-S^J$-p#$Bx8mSxU9Kd46vNHY*+J_Nac4)%K4 z@N--nbnNw%xSLZK++?oe%Z^Wi%mLMJQvZ6@MHzo`2dDt^10`)ld&Jc=^{BT7(Nd6( z)Iw&6+7QS~m^vXexlt~91n0d~47O@Y-i}yM7wR5LUkvHnm$=Mz8OMyxX>vYZE6 zPo+h_#>W>BK~Gb-erwy99OLKJpwe4LTFpl4A~F>>rBnqxwtqa+3b@Y1+By4mq?8XC zXSE_dy;|wSieD-8)1XtTr=+H=fS6-rg#?74AagPFhp7Q3SX-~3Joip&{A8^1YAL0{a5Z7+5UOCK zxT^}DI@gzy`P-M4nn{#?&G6~hr>_u`nd*jva+Q`Q-UDZe9IZtWUp0w?tXaJhpe_w_ zCKZjjnvffjvj>+eL6LV2dtPelOdJ)|bJp_EMn9y3fed&DvIxELgNb)EdLUDX7Iqo9 zuc)DIQ^)ZZKgnRAkj3|@vC-w__m_p3ZPmDK^TgcRT+PYxVq+V~OsWD(^r^b7?3pTc zL);{Tv^sKv(C@?#SRkm&I{dV{Kg4BwUmh1JC3-!x72kK1<^$K|5l(vH?PjK)2bo^a zVZ>^M3r0GJswr@bh-Wa8F?v+n~0u`K=>Y^HWWx{tx6=GOZ6l_mJ17h|>CM z(d7P8D@cNv=%THYpNGiZ0|<_o%z`+thD5Bf#=9|jG15Q3I^w@E@r%d2=>zNIvKHb{ zQp({d)vdA)O_NtI9d2X@Dss|v;o94sMm6yx-jb5DOf8@hb(TOd$(#h$ZENbCM!+iw z;K=dSsNKIea1%cWgKw@%@|aT{QXaviWaa&)v9NF(0CV2jY6$<;n-RRxW5=K(F38)9 z6Nh7btDYVtGUB@I(`L;mF*$ikQo-^Z(?LkbcS21h1wKrGqAgw`VwgPteY=*bssr#} zxGUSgei1!>{B33prA)`K@|9?w22|oP{3^l7y-O)oz;U@P_*n;}@_tUfimMo}+CObi zydh->gvZgZUVt{X=4uZLAw_7|q%F@t2$t~n2FZ|aN^&wCJ-u-Z_-8&5GJj)_Y;kS2lcb8cK@Pxqm>xPR}LRl_#_QZFKd+1Qg zNZ&UUMA6R#j{xAczy#z22dgtmeu|(oU4Mh^T+L#mcQcs;vd`AWX1zZtuuJVYcyMH7 z1PX_>-VBM~`fQIE9Ifhb)2#o*r4Z^*;XjzIwzde%@Vj8FH!!m7^r#)?n>Gdx;09hA2CZX-%h zDT$G>F=4-}(mZEeDzW~mM8MS;H@|ug@D3n9Y^KYWuP^_sowU~2FWuZ+9`uD5RYf@* zR$2NtVtWpyivsB06;d@i*`IF)vn0cz67%U#;$!j2p*qqnDDd15A3o&enM+9pUlD6VZk`qE z-*g$wMx>5?B`07u-e3v%UR0$r zKXa%ZPET|>J6K{DhmUf74oj^XMS|2OiZo5s`-$Hwu%$I~UemZobs}U%JDHuI66;g* zXv%{!Lq6rn+r=FNaXfN)}xPx-4C^Z4S@ilB>p^hvJ zaF|F0e)1*u#b=%O*pPT}6=$7vE@Yto4@rcT_6d&bOd;Zyp{aQ%cxzD}rVWVGprWIA zD$1DNZJcIRuTZRIWcU;clki%N$Chu`nAu_`pQ7U0OJn^(GzCME6m8Cv5Z1dv<4?0i(oy@Wv7*h8%5SgFu93;Y%ShvL_PAkr z(YjCZ0ya-Pn}doHHvdS-6}Ux8{jul{mAhr|Xeqv4w+qh{#>9X&6;_-SFnls%k7H0q z^!y1nQdC-AelV_xC!WeuZ4XXt-S#QrC5}2G^7pk~lC4J+Aa6VuIye*VkAqMbe;NxR zST6LG!PnN;tJzbDr61>n8_}y|c zUQVE)!9m-&B5m#7;(rw!^R3cdg8)8Ts&Nf_al)#6a=}tK@@dok*0*>ys6B*+=x<`3 z+3U^@;lc6W=|8cnjoTn8!7~fg?aQ$^tQ`I6o6GN;^KSXW*P1v!anUfn%9*1?ho-6K zhEkzvn<2gWO#(U&{GFAg=*JTAdBsK2;lq#NLN9sT5v|zp&}7`zzpMwq7ST07D=*PG zT6g1@JzYQw8TKd+o-zI;^s`4Y;5qrqc?tR5aO%+qYOmM#rxNt^pWz72`oRhpR@znz zcrrg%J)A<0=7A)M7>mD_@;>}rEgUoNz3O>3EIjX+rF3(OSQuryldNcK!2YFeJ8=t# zhN@CL_#b|0mcjEMY=>7QY0mRLdvfr=egOc0rZ^(CIn4c}wQ#*pB#ZYnp z1AXdaH@0k05ew{|itDKcjC+a?(&r-Wy4#W#)DH@uO=;ZJ zkByE7y)m@2voka_eEat8NN*Ln-*+w~?6ViGZ!TWQ*s(@~s-KSB3rkkGk+lLK$`X4q zrqMm74=YX@W*wq+elUWZo{^DJRFtB_%r>$^IXsJ59=Dl)a2%h8$s^Z%UUoL7NW@xU zo6Lp`CXoQg!NMX$fB5|D>LbUl;QV~yI+uEGsDTT!HFjQbw?^*)?bZ|c=ww3T8kOsc zS4$r~1H&dxIrD=TGiXIV%h#}w2%4U-q`)T{yea> zW|wFqYZ?U07n5XXx81V@R#hl!4& ziMs(aL15khV1fZ#lPUi=Zun>H)~)fvrCN7mD+>4mQEJqQ{`cwdY}7#xcl{D+1lS`T zs~52zG0gw{*Hqc`ZV-1wDJUPzFsoEZ4927og>{DPh*{sY6#j~*p9n#F&|Wt~nJU6> z<@(BKSMF|B$m$~(q`&((QVCGDl!r&H-=GQ{85d?YyJotVv7-_}NfMHkhW?Io9(N_VThE^u+r(j`?i+gpQcpyz~0ffa=s8d8UKl46w#$klzEMaWcFMfmh-YZmz%w|TaWu@_KAq;VXS zjaX#6U6L5u3^Bf4tk1WNMW52q)Y8&YSHEuo%6LvW49wKTKGM7tZdLVqnyb`mAv7;e zSwH$6TH1-8%q1CQF8alip)# z`~U)NtP?PH_3?+kNRUJvNs#UDC?~0uZX`berd%NB5WcENVW}5!*Ccpv83gN243YvS zaO&_hKYIl~$pi-qxJ|3H3g5tK;nced>lg_7FWtfB9Z7W8E3J>C5)*~69ezTG79g~s zjVsEYb_Zg22vvY4WtKMFffx!h2{>`Vrb%*351N00~Rxhz551`tk2%!_l$iT@qde+s_% za1lbfDoqS9(np1GFJkj;5AQ!}0G`HoJm6%P&czZ5DjEd+&j!XRE)v%RECSFYNv`)8 zi6}ahLOtOR{KDUZ39z=@8VmhUkXKdff{;=76ppm|^$;9d1lZMyTWN86`n>n=QOF(< z5fRNddP9lVhE5LqFwmi{td@u1zMUno(5E2KK+^g^F_VN1mfty%e@ZWlaC)TtCsio z`u#cvOZesH&*o;^Wh=wbQe++XuB)ndqmy&g+U*g#hQB?k}VWl6FV^2MFc@AZX#X>t}Co zZ+2JMwTPRTnDF%UWH)Gf`-q91y|%nOiQ63a>F-Dq_A(irl}uFCizKUS43ML+pAE2E zC{~&enYk@C`Rb~us62mcVs373X2xeTeRy+yY0)w5dG*&I{dl=Ol(uFa{Ei{xmbQ68 zzs)SK2rEW8jqAwUtKNEu0i~EI*&FFsdXvdYa6|rd{%31o=#5H2K>UUA=Jp)Ol(H~_`^2hPoF_G>)bPnu!mLOdF(<4uJW6&bGR~I89Gae}qQ~lmpy6Jc^2nzxlv8dwELM*13Y!7{ESh;NVp;E2*IbA%~=_zk|RyV0-%uIobltI7-51bq~T z(3}ZC;dIy+TmzPZjbb4VYoYMcnW8=#>{qC8^KQ21C{@q?QE%Mz{VB4?AJ4GYN~nW+ znN0VYFB;hsJhI7t19KGfdMUNGzYjNi@TC!9PviPie!(G)$8A?zGdVUEY)ZVe11+J$ zq%>1^JY|{cD~m_@bx~2+r>pX3D;3e`P&TWsD6)K^M~_0m=3)l7H8hCuHkd2=c|+#? zU3>@h!%JhR$?@U0s_)+Yod4lmYS29B8_5J42UTKCy&|m=fcA%Nm|P)C`fD! zG`#vGJa`x{C5}AyMX8=q$RV3hgwgPUKsSQZJe*wmTth-FEj9JAds`63zxojAjS?#Z zg?rxLojUZXvV;*<9nyeWjNlBH>$*8?*!!8*f9T`? zFC^igrktrd^$o5l;M_zie_;z7IOy)qHbdogwCXw!fgegTV#(aZ?{waq;HOO$#FZiKc|M zGp71jXj~%P(VLU3v#V<|idLA$^J%suY{H(VdIgzU`QwM6Rf+HA*)%1W3A)wf|InYD zhWXdKAMEe!SpP2pN;|hWAkd;bllFkT0H&w+XK&8j-kv>~moUem*^dJ74KtPWHT24< z40-1ttV?p@)7WjUx~crXG13?T2Cam?2UGva+1$)bLHE=suD9GjBT`nK9WF5=w?u); z$aHUSZvz^@h99`Q@dAy%7{b5VnNEgk*}vQVN29WsVZZb(bWb|^si@E2Pf8t~ov7h0 z+fjxD1O$?m(vrJ8yu8E%?$c$4o99Qz$K?xEmKu`XD$Tby-jm`GZS9^etE?#x|H{@1 z1zqc=GESZ2`hzBKK+~8t?-H62Q&J9Q=TaIdVC~YfChSI8iM}R76wXLLZHfO;(eKl= z{6$5~(C>|Tl)wA_eE1%X%*S{2)>H6)m3*XDiH;%O@iI+$gfW|c>uu|q|Lx@x4msbq zOhfM7-T5Dt7EMujX!OxP+(Um)CQjT}8%W{~jh=X0IupC^_RnA}3jWr6oi5=GAjMv5 zp`xN%h)y?~_C4YKtE2Od-4axZG@#W6WYl(GQL=4^6fF1+b>5kQl!rvrZ8E$Zyfm&@wB^J}mwjSdn>~l)C;^l!R;2`txq1l$+&RF9{*NO%xe- z$nOD@jBokF=EnZOXvcqAGmhwso+KYk?zl7le4o5W%c)gDHbDq=oehgb+?=0T2lT7o zd&Z=zt81f@#3jXpFWGTresgthK|7Gbe@A`+3N`Aw(d#t8GjS7Cq|NLk3S!Q!SxZFD z#XOH5Dh0iLQ+)bR3iUlF%jV9`?a7!5IK^?+rR-gbAFo?(0I6kF10B$=Hq%srO<2-= zT*1V9H1IWvn8#`yosd#dd|FbKSGGnf^2y(c>FLC4u2Eo*IM~?lu&xKx3yrKK&sLg%+orStil2TQ6ktS&Vi>`1Ad1mF`t!B;%S;M@K9dg?e9A5j#6MOQ za$o4`-S*TM#={)enxZwsn4QZVJHqgoho|oRR6ME-VstR9MfOg_I+Px zTR5fKy_AzLWRN?E;BpTI?z7}q$(bv8yw6y4km~t-m;`hiw&FGi8r<9Pu z6r^7oa2!$u-AhVwRclHO}x-i4d(+AAC#Z3 z<>}bl*x^M2`fW)1wNWd-*(yspIXMs-EO3sB)vb{Fg7@&Jrgm2qz6=*f^SozPe*VV( zIciQ&s5w{8M8iKCq*vCjZdr$v_)OUM=VXolL&Q8p4ZdSyO!-lTh3m`8IL^|NP1sq> zV!IqVdo`g3x!)Xs3@N{w@!_LKm47;78>?ERgP5CB+8A#aHWND&n9-rO)v-bj#JWX2 z7#_~Jg7?TBxokY%dZiZ^z0wHN`GEnCn|6{Cb)@xnoRr>ffuvVsZZ{BHW8R67j%aFN zYAP@Qx~gL@9O7%VS{1u{uOfJZALxs{e*N00k66AH)O25y7vMoH(ae;?`%ASo!cG>$ za-W+y<@^Lmz&Bt`R+uL2ZPmHXYqv4gn0%j{JC(p2g`$~R^$xM16<YvR?hoT`Oe;*QdZjn6#DfNat~)}b&*S_; z7yUDAMpKOxzFb&`35{X&Gz5lCLCrI6etQE6V*aYsiv{wFU)Y4;XKL;$lB9kxWrK*S z+4twGO69a!G&Ht?;{LxE*4;I>bBc-Yl2W{*1X+Zu+8c~z_~|f>bm`W5 zML#wDeX=z)?RUC8I$M1Vu)u6Z&i5SCrIQoY@E@yXKqMvP#v_d^g8H*+6;_)6!KJWT z{NXGNO7y!b5Xe^%Q9b+|>nU7@lIqhIplOgYo=6U;QuGWe>Nx&vMkA=if&-s?ECZ0wGxNbXZoo~r8VRPK$+#6+qk|K>6Gr5_xjs1I50NAue~d;4RwQSM8Z zscC@`pO~-LobUn9_4&qizn}d?{;FFgk4p~ZhGo$icqxCNr&fYOI*WoxhOYn@!5 zR>-VpZ&|DeT|!6^?rhi?A`=Tp!S2oR+jl)@WG`h;%ox7c-euICh5s{gdivGK>`<>D zroas4NoQnaNXWnw76-FSopAwmLWRQ26_RIYSzNatsl;uzHOcvA_-q%Y`r6O>zHUW7 zCEJ}n6|Kn-zeP>EpTZRo-n-OTp{?;LF(69uJm7A@Y-v0dS@59?Rrq<1rbVS?9F%gj z^an;aQq5Sk5u!)^NurxI=0i2zoS}I3CRun?i+B{&Ue;$lySp?3XxA_5ySvOda0&aK z>apYQObxmG>bZx`vHk6JBP|~VDKP~H9|(Ia$--kq*{Aaa8lPersW#xdp1n19GY>1W zn+;n+H2I}%?TYLh+-?Vn5okOw&wXb)(X5h@zP{d=_?9_9_3ygGEWNUgp>ug9xuu~Y zUBc;|)`#^R8j5?)c`v+=&&2OWPVxgup2OYyOC95FX@VUml*oURPZbwSOlJT&j-!qh zUvNNx21hT#QG%J7yR7nGElw_#4cot3oN|qD<;UGn7{a0(#XwsYVb@&k)8-AqUn!HJ0LWo0q;=|pmJHbA|YOzEB)M{g2uA7y3McZq@MC>`5* zgzC+|9Wup46SdrRc===J>U<0&;`3FOt80A=^9y-A){{ZMRV1@A#2Rs=kADAd;VYaT zapdXJJ}KsCZ}MfPUH|CkvfQ>MJ2PBHHY#aV{ggoB5%RtC;RF;HD0D*AfQ(!pOpH~d zX-IFaJoY|f`&)F?G9Sj{F!;wAX%aS-IP;-VjxViM@@Iw)-#G<1KDwmcm$sRFP;C8) zTLbtA7jnkhx~g_*pjF8U?&&#gYC2vvJ6Nc%Fc8SOxc-Ecf|3ahkiqbx-f3av4Zda7%BfZp!imtK75Gz*@im`czaDI1IS0y@3K9~V zY)%*>1MQs*238h}=0L31`@du`KZ_a01$V{20p+UxeEHoC5FH&I05G++wI3Hy-br2l zan&J?K%>i$|LS>5@53)dpGTY1mCsf{(1~J>NK8-vSdqCJ{(|wPCI!>ui7+G!BO^#u zOxOozW!n*%d9N8q)>`Cb|3^DBv&R2x-`uouz;)Pkmczt=gknnaW^w_A{daXRpycXU zq@$@0id>pep&lo=0ILCEht^UOb5$O6t{G+xLjK%W#T34Q*4EaK+0uaU2-g#N?)`f~ znyCY#S#>cK9QnwE-L@hY(=lhlpA81uEWg|(6(u*$%PaS06jJUWspV+~771RD{;UZ| q7{GTG#u@Lvx$r-~q_A^w%L8L5v}d_W1Yd+fk&{xEgui(g@P7c|0_!sX diff --git a/docs/engine.uml b/docs/engine.uml index 03902fbd..404e8663 100644 --- a/docs/engine.uml +++ b/docs/engine.uml @@ -3,10 +3,22 @@ package "moban" { [engine factory] [jinja2 engine] +[data loader] +[yaml loader] +[json loader] +[file system layer] } [lml] +package "pyfilesystem2" { +[fs] +[tar] +[zip] +[file] +[s3] +} + package "moban-mako" { [mako engine] } @@ -15,9 +27,30 @@ package "moban-haml" { [haml engine] } +package "moremobans/gitfs2" { +[git repo] +} + +package "moremobans/pypifs" { +[python package] +} + +[fs.dropbox] + + [engine factory] -> [lml] : get all engines [lml] <-- [jinja2 engine] : register [lml] <.. [mako engine] : register [lml] <.. [haml engine] : register - +[lml] <.. [yaml loader] : register +[lml] <.. [json loader] : register +[data loader] -> [lml] : get all loaders +[file system layer] -> [fs] : access templates,config,output +[fs] <.. [git repo] +[fs] <.. [python package] +[fs] <.. [tar] +[fs] <.. [zip] +[fs] <.. [file] +[fs] <.. [s3] +[fs] <.. [fs.dropbox] @enduml diff --git a/docs/extension.rst b/docs/extension.rst index b7ea1bc2..eefd0a8a 100644 --- a/docs/extension.rst +++ b/docs/extension.rst @@ -37,7 +37,7 @@ Template engine extension for Moban moban version 0.2 started using `lml`_ to employ loose couple plugins. Other template engines, such as marko, haml can be plugged into moban seamless. -.. image:: engine.png +.. image:: images/engine.svg In order plugin other template engines, it is to write a lml plugin. The following is an example starting point for any template engine. diff --git a/docs/images/engine.svg b/docs/images/engine.svg new file mode 100644 index 00000000..97c6babc --- /dev/null +++ b/docs/images/engine.svg @@ -0,0 +1,69 @@ +mobanpyfilesystem2moban-makomoban-hamlmoremobans/gitfs2moremobans/pypifsengine factoryjinja2 enginedata loaderyaml loaderjson loaderfile system layerfstarzipfiles3mako enginehaml enginegit repopython packagelmlfs.dropboxget all enginesregisterregisterregisterregisterregisterget all loadersaccess templates,config,output \ No newline at end of file diff --git a/docs/level-10-moban-dependency-as-git-repo/.moban.yml b/docs/level-10-moban-dependency-as-git-repo/.moban.yml index 884a8206..fc132401 100644 --- a/docs/level-10-moban-dependency-as-git-repo/.moban.yml +++ b/docs/level-10-moban-dependency-as-git-repo/.moban.yml @@ -1,11 +1,9 @@ -requires: - - https://github.com/moremoban/pypi-mobans configuration: template_dir: - - "repo://pypi-mobans/templates" + - "git://github.com/moremoban/pypi-mobans.git!/templates" - local configuration: config.yml - configuration_dir: "repo://pypi-mobans/config" + configuration_dir: "git://github.com/moremoban/pypi-mobans.git!/config" targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 diff --git a/docs/level-10-moban-dependency-as-git-repo/README.rst b/docs/level-10-moban-dependency-as-git-repo/README.rst index 32e0f568..6b1bc912 100644 --- a/docs/level-10-moban-dependency-as-git-repo/README.rst +++ b/docs/level-10-moban-dependency-as-git-repo/README.rst @@ -1,6 +1,10 @@ level 10: moban dependency as git repo ================================================================================ +.. note:: + + You will need to install gitfs2 + Since the support to have a pypi package as dependency, the moban pro user will find it more useful to have git repo so that the changes to static content could get propagate as it happens using git push and git pull. @@ -11,14 +15,12 @@ is welcome to add or improve this feature. Here are the sample file:: - requires: - - https://github.com/moremoban/pypi-mobans configuration: template_dir: - - "repo://pypi-mobans/templates" + - "git://github.com/moremoban/pypi-mobans.git!/templates" - local configuration: config.yml - configuration_dir: "repo://pypi-mobans/config" + configuration_dir: "git://github.com/moremoban/pypi-mobans.git!/config" targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 @@ -27,16 +29,21 @@ where `requires` lead to a list of pypi packages. And when you refer to it, as in level-9 section, please use "pypi-mobans:" -Alternative syntax when submodule exists +The syntax when submodule exists -------------------------------------------------------------------------------- -The alternative syntax is:: +The sumodule syntax is:: - requires: - - type: git - url: https://github.com/your-git-url - submodule: true - branch: your_choice_or_default_branch_if_not_specified - reference: your_alternative_reference_but_not_used_together_with_branch - ... + configuration: + template_dir: + - "git://github.com/moremoban/pypi-mobans.git?submodule=true&branch=your_choice_or_default_branch_if_not_specified!/templates" + - local + + +If you have reference instead of branch:: + + configuration: + template_dir: + - "git://github.com/moremoban/pypi-mobans.git?submodule=true&reference=your_alternative_reference_but_not_used_together_with_branch!/templates" + - local diff --git a/docs/level-9-moban-dependency-as-pypi-package/.moban.yml b/docs/level-9-moban-dependency-as-pypi-package/.moban.yml index 5618d05f..b6c703cc 100644 --- a/docs/level-9-moban-dependency-as-pypi-package/.moban.yml +++ b/docs/level-9-moban-dependency-as-pypi-package/.moban.yml @@ -1,5 +1,3 @@ -requires: - - pypi-mobans-pkg configuration: template_dir: - "pypi://pypi-mobans-pkg/resources/templates" diff --git a/docs/level-9-moban-dependency-as-pypi-package/README.rst b/docs/level-9-moban-dependency-as-pypi-package/README.rst index 14ccb3c0..6d69f871 100644 --- a/docs/level-9-moban-dependency-as-pypi-package/README.rst +++ b/docs/level-9-moban-dependency-as-pypi-package/README.rst @@ -1,6 +1,10 @@ level 9: moban dependency as pypi package ================================================================================ +.. note:: + + You will need to install pypifs + Why not enable template reuse? Once a template is written somewhere by somebody, as long as it is good and useful, it is always to reuse it, isn't it? DRY principle kicks in. @@ -11,40 +15,23 @@ into a pypi package and distribute it to the world of moban. Here are the sample file:: - requires: - - pypi-mobans-pkg configuration: template_dir: - - "pypi://pypi-mobans-pkg/templates" + - "pypi://pypi-mobans-pkg/resources/templates" configuration: config.yml configuration_dir: "pypi://pypi-mobans-pkg/config" targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 -where `requires` lead to a list of pypi packages. The short syntax is:: - - requires: - - python-package-name - When you refer to it in configuration section, here is the syntax:: configuration: - template_dir: - - "python-package-name:relative-folder-inside-the-package" + - "pypi://python-package-name/relative-folder-inside-the-package" -Note: when you do not have relative directory, please keep semi-colon:: +Note: when you do not have relative directory:: configuration: template_dir: - - "python-package-name:" - -Alternative syntax --------------------------------------------------------------------------------- - -The alternative syntax is:: - - requires: - - type: pypi - name: pypi-mobans - ... + - "pypi://python-package-name" diff --git a/moban/constants.py b/moban/constants.py index 93967b09..a139ffb1 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -130,5 +130,9 @@ ) % TEMPLATE_COPY MESSAGE_DEPRECATE_MOBAN_NOTATION_SINCE_0_6_0 = ( "File path notation is deprecated since 0.6.0! " - + "Please use pypi:// or repo:// notation instead" + + "Please use pypi:// or git:// notation instead" +) +MESSAGE_DEPRECATE_REQUIRES_SINCE_0_6_0 = ( + "'requires' is deprecated since 0.6.0! " + + "Please use pypi:// or git:// notation instead" ) diff --git a/moban/definitions.py b/moban/definitions.py index a78a8f07..982e3e5d 100644 --- a/moban/definitions.py +++ b/moban/definitions.py @@ -1,38 +1,6 @@ from moban import constants -class GitRequire(object): - def __init__( - self, git_url=None, branch=None, submodule=False, reference=None - ): - self.git_url = git_url - self.submodule = submodule - self.branch = branch - self.reference = reference - - def clone_params(self): - clone_params = { - "single_branch": True, - "depth": constants.DEFAULT_CLONE_DEPTH, - } - if self.branch is not None: - clone_params["branch"] = self.branch - elif self.reference is not None: - clone_params["reference"] = self.reference - return clone_params - - def __eq__(self, other): - return ( - self.git_url == other.git_url - and self.submodule == other.submodule - and self.branch == other.branch - and self.reference == other.reference - ) - - def __repr__(self): - return "%s,%s,%s" % (self.git_url, self.branch, self.submodule) - - class TemplateTarget(object): def __init__( self, diff --git a/moban/deprecated.py b/moban/deprecated.py deleted file mode 100644 index 4c8c03a6..00000000 --- a/moban/deprecated.py +++ /dev/null @@ -1,15 +0,0 @@ -from functools import wraps - -from moban import reporter - - -def deprecated(message): - def tags_decorator(func): - @wraps(func) - def func_wrapper(*args, **kwds): - reporter.report_warning_message(message) - return func(*args, **kwds) - - return func_wrapper - - return tags_decorator diff --git a/moban/deprecated/__init__.py b/moban/deprecated/__init__.py new file mode 100644 index 00000000..f78f364c --- /dev/null +++ b/moban/deprecated/__init__.py @@ -0,0 +1,56 @@ +import sys +from functools import wraps + +from moban import reporter, constants + + +def deprecated(message): + def tags_decorator(func): + @wraps(func) + def func_wrapper(*args, **kwds): + reporter.report_warning_message(message) + return func(*args, **kwds) + + return func_wrapper + + return tags_decorator + + +class GitRequire(object): + def __init__( + self, git_url=None, branch=None, submodule=False, reference=None + ): + self.git_url = git_url + self.submodule = submodule + self.branch = branch + self.reference = reference + + def clone_params(self): + clone_params = { + "single_branch": True, + "depth": constants.DEFAULT_CLONE_DEPTH, + } + if self.branch is not None: + clone_params["branch"] = self.branch + elif self.reference is not None: + clone_params["reference"] = self.reference + return clone_params + + def __eq__(self, other): + return ( + self.git_url == other.git_url + and self.submodule == other.submodule + and self.branch == other.branch + and self.reference == other.reference + ) + + def __repr__(self): + return "%s,%s,%s" % (self.git_url, self.branch, self.submodule) + + +def pip_install(packages): + import subprocess + + subprocess.check_call( + [sys.executable, "-m", "pip", "install", " ".join(packages)] + ) diff --git a/moban/repo.py b/moban/deprecated/repo.py similarity index 100% rename from moban/repo.py rename to moban/deprecated/repo.py diff --git a/moban/fs_openers.py b/moban/fs_openers.py deleted file mode 100644 index f22512c4..00000000 --- a/moban/fs_openers.py +++ /dev/null @@ -1,18 +0,0 @@ -import fs -import fs.path -from moban import repo -from fs.osfs import OSFS -from fs.opener import Opener -from fs.opener.registry import registry - - -@registry.install -class RepoFSOpener(Opener): - protocols = ["repo"] - - def open_fs(self, fs_url, parse_result, writeable, create, cwd): - repo_name, _, dir_path = parse_result.resource.partition("/") - actual_repo_path = fs.path.join(repo.get_moban_home(), repo_name) - root_path = fs.path.join(actual_repo_path, dir_path) - osfs = OSFS(root_path=root_path) - return osfs diff --git a/moban/main.py b/moban/main.py index 7ad010cc..75eec3d0 100644 --- a/moban/main.py +++ b/moban/main.py @@ -12,7 +12,6 @@ import logging import argparse -import moban.fs_openers # noqa: F401 from moban import plugins, reporter, constants, mobanfile, exceptions from moban.utils import merge from moban._version import __version__ diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index cae13c74..81696ae2 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -5,10 +5,9 @@ from moban import plugins, reporter, constants, file_system from lml.utils import do_import -from moban.repo import git_clone -from moban.utils import merge, pip_install, verify_the_existence_of_directories -from moban.deprecated import deprecated -from moban.definitions import GitRequire +from moban.utils import merge, verify_the_existence_of_directories +from moban.deprecated import GitRequire, deprecated, pip_install +from moban.deprecated.repo import git_clone from moban.plugins.template import expand_template_directories from moban.mobanfile.targets import parse_targets, extract_group_targets @@ -188,6 +187,7 @@ def extract_target(options): return result +@deprecated(constants.MESSAGE_DEPRECATE_REQUIRES_SINCE_0_6_0) def handle_requires(requires): pypi_pkgs = [] git_repos = [] diff --git a/moban/plugins/template.py b/moban/plugins/template.py index 6649231b..d87f6e73 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -2,10 +2,10 @@ import sys import logging -from moban import repo, utils, reporter, constants, exceptions, file_system +from moban import utils, reporter, constants, exceptions, file_system from lml.plugin import PluginManager from moban.hashstore import HASH_STORE -from moban.deprecated import deprecated +from moban.deprecated import repo, deprecated from moban.buffered_writer import BufferedWriter from moban.plugins.context import Context from moban.plugins.library import LIBRARIES diff --git a/moban/utils.py b/moban/utils.py index e7d36608..23e53f62 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -39,14 +39,6 @@ def mkdir_p(path): raise -def pip_install(packages): - import subprocess - - subprocess.check_call( - [sys.executable, "-m", "pip", "install", " ".join(packages)] - ) - - def verify_the_existence_of_directories(dirs): LOG.debug("Verifying the existence: %s", dirs) if not isinstance(dirs, list): diff --git a/rnd_requirements.txt b/rnd_requirements.txt index efab09e1..7722178c 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,5 +1,6 @@ https://github.com/moremoban/pyfilesystem2/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip https://github.com/moremoban/pypifs/archive/master.zip +https://github.com/moremoban/gitfs2/archive/master.zip diff --git a/tests/test_repo.py b/tests/deprecated/test_repo.py similarity index 98% rename from tests/test_repo.py rename to tests/deprecated/test_repo.py index 0041cc4e..1cfc571b 100644 --- a/tests/test_repo.py +++ b/tests/deprecated/test_repo.py @@ -1,14 +1,14 @@ import fs.path from mock import patch -from moban.repo import ( +from nose.tools import eq_, raises +from moban.deprecated import GitRequire +from moban.exceptions import NoGitCommand +from moban.deprecated.repo import ( git_clone, get_repo_name, get_moban_home, make_sure_git_is_available, ) -from nose.tools import eq_, raises -from moban.exceptions import NoGitCommand -from moban.definitions import GitRequire @patch("appdirs.user_cache_dir", return_value="root") diff --git a/tests/mobanfile/test_mobanfile.py b/tests/mobanfile/test_mobanfile.py index a6009438..f4cb6487 100644 --- a/tests/mobanfile/test_mobanfile.py +++ b/tests/mobanfile/test_mobanfile.py @@ -1,7 +1,8 @@ import fs.path from mock import patch from nose.tools import eq_ -from moban.definitions import GitRequire, TemplateTarget +from moban.deprecated import GitRequire +from moban.definitions import TemplateTarget class TestFinder: diff --git a/tests/regression_tests/level-21-b-copy-templates-into-a-tar/.moban.yml b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/.moban.yml new file mode 100644 index 00000000..c9ceba44 --- /dev/null +++ b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/.moban.yml @@ -0,0 +1,19 @@ +configuration: + template_dir: + - "tar://template-sources.tar" +targets: + - output: "tar://my.tar!/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "tar://my.tar!/target_without_template_type" + template: file_extension_will_trigger.copy + - "tar://my.tar!/target_in_short_form": as_long_as_this_one_has.copy + - output: "tar://my.tar!/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "tar://my.tar!/test-dir" + template: dir-for-copying + template_type: copy + - output: "tar://my.tar!/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy diff --git a/tests/regression_tests/level-21-b-copy-templates-into-a-tar/README.rst b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/README.rst new file mode 100644 index 00000000..533269b5 --- /dev/null +++ b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/README.rst @@ -0,0 +1,83 @@ +Level 21-b: template copying from a tar to a tar +================================================================================ + +In level 15, with `.moban.yml`, you can copy templates to your destination. Now +with similiar moban syntax, let me show how to create a new zip file where +all templates are copied to. + +Explicit syntax:: + + targets: + - output: "tar://your.zip/explicit" + template: template_file + template_type: copy + + +Implicit syntax:: + + targets: + - output: "tar://your.zip/implicit" + template: template_file.copy + + +Shorthand syntax:: + + targets: + - "tar://your.zip/shorthand": template_file.copy + + +No implicit nor short hand syntax for the following directory copying unless +you take a look at `force-template-type`. When you read +`level-17-force-template-type-from-moban-file/README.rst`, you will find +out more. + + +Directory copying syntax:: + + + targets: + - output: "tar://your.zip/dest-dir" + template: source-dir + template_type: copy + + +Recursive directory copying syntax:: + + + targets: + - output: "tar://your.zip/dest-dir" + template: source-dir/** + template_type: copy + + +Evaluation +-------------------------------------------------------------------------------- + +Here is example moban file for copying:: + + configuration: + template_dir: + - "tar://template-sources.tar" + targets: + - output: "tar://my.tar/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "tar://my.tar/target_without_template_type" + template: file_extension_will_trigger.copy + - "tar://my.tar/target_in_short_form": as_long_as_this_one_has.copy + - output: "tar://my.tar/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "tar://my.tar/test-dir" + template: dir-for-copying + template_type: copy + - output: "tar://my.tar/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy + + +template copy does: + + +#. copies any template inside pre-declared template directory to anywhere. moban will create directory if needed. +#. copies any directory to anywhere. If "**" is followed, moban attempts to do recursive copying. diff --git a/docs/level-21-copy-templates-into-an-alien-file-system/template-sources.tar b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/template-sources.tar similarity index 100% rename from docs/level-21-copy-templates-into-an-alien-file-system/template-sources.tar rename to tests/regression_tests/level-21-b-copy-templates-into-a-tar/template-sources.tar diff --git a/tests/regression_tests/level-21-c-copy-templates-from-a-tar/.moban.yml b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/.moban.yml new file mode 100644 index 00000000..d3b82590 --- /dev/null +++ b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/.moban.yml @@ -0,0 +1,19 @@ +configuration: + template_dir: + - "tar://template-sources.tar" +targets: + - output: "zip://my.zip!/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip!/target_without_template_type" + template: file_extension_will_trigger.copy + - "zip://my.zip!/target_in_short_form": as_long_as_this_one_has.copy + - output: "zip://my.zip!/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip!/test-dir" + template: dir-for-copying + template_type: copy + - output: "zip://my.zip!/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy diff --git a/tests/regression_tests/level-21-c-copy-templates-from-a-tar/README.rst b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/README.rst new file mode 100644 index 00000000..79ab39f2 --- /dev/null +++ b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/README.rst @@ -0,0 +1,24 @@ +Level 21-c: template copying from a tar to a zip +================================================================================ + +Here is another set of regression tests file for :: + + configuration: + template_dir: + - "tar://template-sources.tar" + targets: + - output: "zip://my.zip!/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip!/target_without_template_type" + template: file_extension_will_trigger.copy + - "zip://my.zip!/target_in_short_form": as_long_as_this_one_has.copy + - output: "zip://my.zip!/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip!/test-dir" + template: dir-for-copying + template_type: copy + - output: "zip://my.zip!/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy diff --git a/tests/regression_tests/level-21-c-copy-templates-from-a-tar/template-sources.tar b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/template-sources.tar new file mode 100644 index 0000000000000000000000000000000000000000..2f13227fd560f0e893e00691f01d6332def4f280 GIT binary patch literal 10752 zcmeHNOOM(x5a!%p;TzHlg5x}pdac?^5A8pQ3>kuVLyT;PUH*N?z!I^n5*|sq-8u)H z#Ls*)-^_T7HyLz(QA(##LnC0_AvRkqYU&T)VLuWtQq%H7Re6(CsPRIov4VN2#RkT6x!#H*Yfmu{k1ORL z2R?K1??X|%$wK5X7RA`WYMpZfV=YTH2aQURr=UFZV(wfB%0CYGe1D(+K0D<jv8S*e3onKR)HZ zKSc}R_}>ot^`i(wgeJd!5+liyZ-8*3mr|ikkZLs|v(gZ=gq?-3O=dubs2Bp$A-3ar zzVBq;ZHEM23=~?(VnnngT-zQ-i7W#}zR3bB%@3)bxk|qmnJ8ei8_agxJ$lXt)eEbG z+A98QJ?uNK_WJlQ3}YAnAMxw*W*`40N9HEaBF~cpXIZ?d|6R)c zb!;2@Rd$@4#uV4ti!xBuL#L<_sV3aCc@vp4+r(TdZy&03gZwcF@yM`Ts}2{Ltn9 o`Vjvh#q-~4yZ05(ZS3KH^=9($;M_qkl5mqa1RMemfqoJA14BUqTL1t6 literal 0 HcmV?d00001 diff --git a/tests/test_definitions.py b/tests/test_definitions.py index 75d2bc60..98a78741 100644 --- a/tests/test_definitions.py +++ b/tests/test_definitions.py @@ -1,5 +1,6 @@ from nose.tools import eq_ -from moban.definitions import GitRequire, TemplateTarget +from moban.deprecated import GitRequire +from moban.definitions import TemplateTarget def test_git_require_repr(): diff --git a/tests/test_engine.py b/tests/test_engine.py index e04dd546..1739db21 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -31,7 +31,7 @@ def test_expand_pypi_dir(): assert os.path.exists(directory) -@patch("moban.repo.get_moban_home", return_value=USER_HOME) +@patch("moban.deprecated.repo.get_moban_home", return_value=USER_HOME) @patch("moban.file_system.exists", return_value=True) def test_expand_repo_dir(_, __): dirs = list(expand_template_directories("git_repo:template")) diff --git a/tests/test_regression.py b/tests/test_regression.py index 37b771d4..1136669f 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -5,8 +5,10 @@ import fs.path from mock import patch +from moban import file_system from moban.main import main from nose.tools import eq_ +from fs.opener.parse import parse_fs_url def custom_dedent(long_texts): @@ -30,6 +32,48 @@ def test_coping_binary_file(self): "regression-test.png", ) + def test_level_21_copy_templates_into_tars(self): + expected = "test file\n" + + folder = "level-21-b-copy-templates-into-a-tar" + criterias = [ + ["tar://my.tar!/simple.file", expected], + [ + "tar://my.tar!/target_without_template_type", + "file extension will trigger copy engine\n", + ], + [ + "tar://my.tar!/target_in_short_form", + ( + "it is OK to have a short form, " + + "but the file to be 'copied' shall have 'copy' extension, " + + "so as to trigger ContentForwardEngine, 'copy' engine.\n" + ), + ], + ] + self._raw_moban_with_fs2(["moban"], folder, criterias) + + def test_level_21_copy_templates_from_tars(self): + expected = "test file\n" + + folder = "level-21-c-copy-templates-from-a-tar" + criterias = [ + ["zip://my.zip!/simple.file", expected], + [ + "zip://my.zip!/target_without_template_type", + "file extension will trigger copy engine\n", + ], + [ + "zip://my.zip!/target_in_short_form", + ( + "it is OK to have a short form, " + + "but the file to be 'copied' shall have 'copy' extension, " + + "so as to trigger ContentForwardEngine, 'copy' engine.\n" + ), + ], + ] + self._raw_moban_with_fs2(["moban"], folder, criterias) + def _raw_moban(self, args, folder, expected, output): base_dir = fs.path.join("tests", "regression_tests") os.chdir(fs.path.join(base_dir, folder)) @@ -39,6 +83,26 @@ def _raw_moban(self, args, folder, expected, output): os.unlink(output) assert status + def _raw_moban_with_fs(self, args, folder, expected, output): + base_dir = fs.path.join("tests", "regression_tests") + os.chdir(fs.path.join(base_dir, folder)) + with patch.object(sys, "argv", args): + main() + _verify_content_with_fs(output, expected) + result = parse_fs_url(output) + os.unlink(result.resource) # delete the zip file + + def _raw_moban_with_fs2(self, args, folder, criterias): + base_dir = fs.path.join("tests", "regression_tests") + os.chdir(fs.path.join(base_dir, folder)) + with patch.object(sys, "argv", args): + main() + + for output, expected in criterias: + _verify_content_with_fs(output, expected) + result = parse_fs_url(output) + os.unlink(result.resource) # delete the zip file + def tearDown(self): if os.path.exists(".moban.hashes"): os.unlink(".moban.hashes") @@ -49,3 +113,8 @@ def _verify_content(file_name, expected): with open(file_name, "r") as f: content = f.read() eq_(content, expected) + + +def _verify_content_with_fs(file_name, expected): + content = file_system.read_unicode(file_name) + eq_(content, expected) diff --git a/tests/test_utils.py b/tests/test_utils.py index 156fcfc6..ab395fe6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -15,7 +15,7 @@ def test_mkdir_p(): @patch("subprocess.check_call") def test_pip_install(fake_check_all): import sys - from moban.utils import pip_install + from moban.deprecated import pip_install pip_install(["package1", "package2"]) fake_check_all.assert_called_with( From 96aa2f9ec8e61270e9e541eba19aa6699b338a96 Mon Sep 17 00:00:00 2001 From: jaska Date: Sun, 18 Aug 2019 08:06:23 +0100 Subject: [PATCH 156/167] Fix any other regressions (#313) * :fire: remove appdirs and gitpython from moban dependency list * :books: add a use case: template from a s3 bucket * :sparkles: add -d for development tracing. fix #185 * :tractor: do not log the exception but as info * :books: update documenation * :bug: moban -t url -c url -o moban.output failes * :microscope: more tests * :shirt: fix format * :books: update change log and readme * :fire: remove unused imports * :microscope: more tests * :shirt: fix coding style * :books: update package url link * :green_heart: make travis tests pass * :green_heart: make windows tests pass. an osfs was passed to multifs and the invalid characters on windows are different from the ones on linx. hence pypifs and gitfs2 url are seen as invalid on windows. https://github.com/PyFilesystem/pyfilesystem2/pull/347 * :fire: remove adapter folder * :bug: fix build failure on opensuse. resolves #275 * :sparkles: use released version of pypifs * :newspaper: add pypifs as test dependency --- .moban.cd/changelog.yml | 2 + .moban.cd/moban.yml | 2 - .travis.yml | 2 +- CHANGELOG.rst | 4 + README.rst | 128 ++++++++++-------- docs/extension.rst | 52 +++++++ min_requirements.txt | 2 - moban/adapter/__init__.py | 0 moban/constants.py | 1 + moban/file_system.py | 2 +- moban/jinja2/engine.py | 6 +- moban/main.py | 14 ++ moban/mobanfile/templates.py | 1 + moban/plugins/template.py | 15 +- requirements.txt | 2 - rnd_requirements.txt | 1 - setup.py | 2 - tests/fixtures/.moban-version-1.0.yml | 1 - tests/fixtures/.moban-version-1234.yml | 3 +- .../test_command_line_options.py | 58 ++++++++ tests/requirements.txt | 1 + tests/test_buffered_writer.py | 45 ++++++ 22 files changed, 266 insertions(+), 78 deletions(-) delete mode 100644 moban/adapter/__init__.py create mode 100644 tests/test_buffered_writer.py diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 7d51f365..87fcf204 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -5,6 +5,8 @@ releases: - action: Updated details: - "`#205`: support `pyFilesystem2 `_" + - "`#185`: -d will enable moban application logging for development" + - "`#275`: fix moban 0.4.5 test failures on openSUSE Tumbleweed" date: tba version: 0.6.0 - changes: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 73e1b50c..52eedfdb 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -26,8 +26,6 @@ dependencies: - lml>=0.0.9 - appdirs>=1.4.3 - crayons>= 0.1.0 - - GitPython>=2.0.0 - - git-url-parse>=1.2.2 - fs>=2.4.6 - jinja2-fsloader>=0.2.0 description: Yet another jinja2 cli command for static text generation diff --git a/.travis.yml b/.travis.yml index d28e28d4..65934ee3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,9 +15,9 @@ env: - MINREQ=0 stages: - - test - lint - moban + - test .disable_global: &disable_global addons: false diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eee86ed8..aae78ac8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,10 @@ Updated #. `#205 `_: support `pyFilesystem2 `_ +#. `#185 `_: -d will enable moban + application logging for development +#. `#275 `_: fix moban 0.4.5 test + failures on openSUSE Tumbleweed 0.5.0 - 14.07.2019 -------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index cbb1034c..48ed575f 100644 --- a/README.rst +++ b/README.rst @@ -35,6 +35,11 @@ any location: zip, git, pypi package, s3, etc. Please look at our issues. We have many more template engines and data format on the road map. +Documentation +================================================================================= + +All use cases are documented `here `_ + Installation ================================================================================ You can install it via pip: @@ -95,9 +100,73 @@ moban.output will contain:: Please note that data.yml will take precedence over environment variables. -`the tutorial`_ has more use cases. -.. _the tutorial: http://moban.readthedocs.org/en/latest/#tutorial +Work with files in a git repo +================================================================================ + +Please install `gitfs2 `_:: + + $ pip install gitfs2 + + +And then you can do the following: + +.. code-block:: bash + + $ moban -t 'git://github.com/moremoban/pypi-mobans.git!/templates/_version.py.jj2' \ + -c 'git://github.com/moremoban/pypi-mobans.git!/config/data.yml' \ + -o _version.py + Info: Found repo in /Users/jaska/Library/Caches/gitfs2/repos/pypi-mobans + Templating git://github.com/moremoban/pypi-mobans.git!/templates/_version.py.jj2 to _version.py + Templated 1 file. + $ cat _version.py + __version__ = "0.1.1rc3" + __author__ = "C.W." + + +Work with files in a python package +================================================================================ + +Please install `pypifs `_:: + + $ pip install pypifs + + +And then you can do the following: + +.. code-block:: bash + + $ moban -t 'pypi://pypi-mobans-pkg/resources/templates/_version.py.jj2' \ + -c 'pypi://pypi-mobans-pkg/resources/config/data.yml' \ + -o _version.py + Collecting pypi-mobans-pkg + .... + Installing collected packages: pypi-mobans-pkg + Successfully installed pypi-mobans-pkg-0.0.7 + Templating pypi://pypi-mobans-pkg/resources/templates/_version.py.jj2 to _version.py + Templated 1 file. + $ cat _version.py + __version__ = "0.1.1rc3" + __author__ = "C.W." + +Work with S3 and other cloud based file systems +================================================================================ + +Please install `fs-s3fs `_:: + + $ pip install fs-s3fs + +.. code-block:: bash + + $ moban -c s3://${client_id}:${client_secrect}@moremoban/s3data.yml \ + -o 'zip://my.zip!/moban.output' {{hello}} + $ unzip my.zip + $ cat moban.output + world + +Where the configuration sits in a s3 bucket, the output is a file in a zip. The content of s3data.yaml is:: + + hello: world Usage @@ -151,58 +220,3 @@ With `--exit-code`: - 0 : no changes - 1 : has changes - 2 : error occured - -Built-in Filters -================================================================================ - -split_length --------------------------------------------------------------------------------- - -It breaks down the given string into a fixed length paragraph. Here is the syntax:: - - {% for line in your_string | split_length(your_line_with) %} - {{line}} - {% endfor %} - -It is used to keep changelog formatted in -`CHANGELOG.rst.jj2 in pypi-mobans project `_ - -github_expand --------------------------------------------------------------------------------- - -It expands simple hashtags into github issues. Here is the syntax:: - - {{ your_github_string | github_expand }} - - -It makes it easy to mention github reference in change log in all projects. Here is -the place it is applied: -`CHANGELOG.rst.jj2 in pypi-mobans project `_ - - -Here is Grammar in the changelog.yml:: - - =============== ============================== - Syntax Meaning - =============== ============================== - `#1` moban issues 1 - `PR#1` moban pull request 1 - `pyexcel#1` other project issues 1 - `pyexcel#PR#1` other project pulll request 1 - =============== ============================== - -More details can be found in `moban's changelog.yml `_ - -`repr` --------------------------------------------------------------------------------- - -Returns a single quoted string in the templated file - - -Built-in Tests -================================================================================ - -`exists` --------------------------------------------------------------------------------- - -Test if a file exists or not diff --git a/docs/extension.rst b/docs/extension.rst index eefd0a8a..1cf2d9d9 100644 --- a/docs/extension.rst +++ b/docs/extension.rst @@ -16,12 +16,64 @@ Jinja2 Filter .. literalinclude:: ../moban/filters/repr.py +split_length +-------------------------------------------------------------------------------- + +It breaks down the given string into a fixed length paragraph. Here is the syntax:: + + {% for line in your_string | split_length(your_line_with) %} + {{line}} + {% endfor %} + +It is used to keep changelog formatted in +`CHANGELOG.rst.jj2 in pypi-mobans project `_ + +github_expand +-------------------------------------------------------------------------------- + +It expands simple hashtags into github issues. Here is the syntax:: + + {{ your_github_string | github_expand }} + + +It makes it easy to mention github reference in change log in all projects. Here is +the place it is applied: +`CHANGELOG.rst.jj2 in pypi-mobans project `_ + + +Here is Grammar in the changelog.yml:: + + =============== ============================== + Syntax Meaning + =============== ============================== + `#1` moban issues 1 + `PR#1` moban pull request 1 + `pyexcel#1` other project issues 1 + `pyexcel#PR#1` other project pulll request 1 + =============== ============================== + +More details can be found in `moban's changelog.yml `_ + +`repr` +-------------------------------------------------------------------------------- + +Returns a single quoted string in the templated file + + +Built-in Tests +================================================================================ Jinja2 Test ******************* .. literalinclude:: ../moban/tests/files.py + +`exists` +-------------------------------------------------------------------------------- + +Test if a file exists or not + Jinja2 Globals ******************* diff --git a/min_requirements.txt b/min_requirements.txt index 802131e9..6c8b55b5 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -6,7 +6,5 @@ jinja2==2.7.1 lml==0.0.9 appdirs==1.4.3 crayons== 0.1.0 -GitPython==2.0.0 -git-url-parse==1.2.2 fs==2.4.6 jinja2-fsloader==0.2.0 diff --git a/moban/adapter/__init__.py b/moban/adapter/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/moban/constants.py b/moban/constants.py index a139ffb1..169cbe15 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -70,6 +70,7 @@ LABEL_DEST = "destination" LABEL_FORCE_TEMPLATE_TYPE = "force_template_type" LABEL_TEMPLATE_TYPES = "template_types" +LABEL_DEBUG = "debug" # error messages ERROR_DATA_FILE_NOT_FOUND = "Both %s and %s does not exist" diff --git a/moban/file_system.py b/moban/file_system.py index 9f8a9001..f08554f4 100644 --- a/moban/file_system.py +++ b/moban/file_system.py @@ -40,7 +40,7 @@ def wrapper(*args, **kwds): message = "Failed to open %s" % args[0] LOG.debug(message) reporter.report_error_message(message) - raise + raise exceptions.FileNotFound(args[0]) return wrapper diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index eea38428..eadf6df3 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -2,7 +2,7 @@ import logging from importlib import import_module -from moban import constants, exceptions +from moban import constants, file_system from jinja2 import Template, Environment from lml.loader import scan_plugins_regex from lml.plugin import PluginInfo, PluginManager @@ -79,6 +79,7 @@ def __init__(self, template_fs, options=None): extension for extension in JINJA2_THIRD_PARTY_EXTENSIONS ], # get a copy of this global variable ) + self.template_loader = template_loader if options: if "extensions" in options: extensions = options.pop("extensions") @@ -117,7 +118,8 @@ def get_template(self, template_file): try: template = self.jj2_environment.get_template(template_file) except TemplateNotFound: - raise exceptions.FileNotFound("%s does not exist" % template_file) + content = file_system.read_unicode(template_file) + return Template(content) return template def get_template_from_string(self, string): diff --git a/moban/main.py b/moban/main.py index 75eec3d0..c5950400 100644 --- a/moban/main.py +++ b/moban/main.py @@ -11,6 +11,7 @@ import sys import logging import argparse +import logging.config from moban import plugins, reporter, constants, mobanfile, exceptions from moban.utils import merge @@ -28,6 +29,12 @@ def main(): parser = create_parser() options = vars(parser.parse_args()) HASH_STORE.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE] + if options[constants.LABEL_DEBUG]: + logging.basicConfig( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + level=logging.DEBUG, + ) + moban_file = options[constants.LABEL_MOBANFILE] load_engine_factory_and_engines() # Error: jinja2 if removed if moban_file is None: @@ -126,6 +133,13 @@ def create_parser(): action="version", version="%(prog)s {v}".format(v=__version__), ) + parser.add_argument( + "-d", + action="store_true", + dest=constants.LABEL_DEBUG, + default=False, + help="to show debug trace", + ) return parser diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 80e3f6c5..73240e38 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -7,6 +7,7 @@ def handle_template(template_file, output, template_dirs): log.info("handling %s" % template_file) + template_file = file_system.to_unicode(template_file) multi_fs = file_system.get_multi_fs(template_dirs) if template_file.endswith("**"): diff --git a/moban/plugins/template.py b/moban/plugins/template.py index d87f6e73..d1b9fae0 100644 --- a/moban/plugins/template.py +++ b/moban/plugins/template.py @@ -3,6 +3,7 @@ import logging from moban import utils, reporter, constants, exceptions, file_system +from fs.errors import ResourceNotFound from lml.plugin import PluginManager from moban.hashstore import HASH_STORE from moban.deprecated import repo, deprecated @@ -88,9 +89,12 @@ def render_to_file(self, template_file, data_file, output_file): template_file = file_system.to_unicode(template_file) data = self.context.get_data(data_file) template = self.engine.get_template(template_file) - template_abs_path = self.template_fs.geturl( - template_file, purpose="fs" - ) + try: + template_abs_path = self.template_fs.geturl( + template_file, purpose="fs" + ) + except ResourceNotFound: + template_abs_path = template_file flag = self.apply_template( template_abs_path, template, data, output_file @@ -135,8 +139,9 @@ def apply_template(self, template_abs_path, template, data, output_file): template_abs_path, output_file ) return flag - except exceptions.FileNotFound as e: - log.exception(e) + except exceptions.FileNotFound: + # the template is a string from command line + log.info("{} is not a file".format(template_abs_path)) self.buffered_writer.write_file_out(output_file, rendered_content) return True diff --git a/requirements.txt b/requirements.txt index b8b77a79..ff6ed355 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,5 @@ jinja2>=2.7.1 lml>=0.0.9 appdirs>=1.4.3 crayons>= 0.1.0 -GitPython>=2.0.0 -git-url-parse>=1.2.2 fs>=2.4.6 jinja2-fsloader>=0.2.0 diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 7722178c..baf71891 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,6 +1,5 @@ https://github.com/moremoban/pyfilesystem2/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip -https://github.com/moremoban/pypifs/archive/master.zip https://github.com/moremoban/gitfs2/archive/master.zip diff --git a/setup.py b/setup.py index 33335d93..94b3dc73 100644 --- a/setup.py +++ b/setup.py @@ -80,8 +80,6 @@ "lml>=0.0.9", "appdirs>=1.4.3", "crayons>= 0.1.0", - "GitPython>=2.0.0", - "git-url-parse>=1.2.2", "fs>=2.4.6", "jinja2-fsloader>=0.2.0", ] diff --git a/tests/fixtures/.moban-version-1.0.yml b/tests/fixtures/.moban-version-1.0.yml index 0e51a764..04ca9eb9 100644 --- a/tests/fixtures/.moban-version-1.0.yml +++ b/tests/fixtures/.moban-version-1.0.yml @@ -2,5 +2,4 @@ version: 1.0 configuration: configuration_dir: "commons/config" template_dir: - - "commons/templates" - ".moban.d" diff --git a/tests/fixtures/.moban-version-1234.yml b/tests/fixtures/.moban-version-1234.yml index 4c265181..73d0d55d 100644 --- a/tests/fixtures/.moban-version-1234.yml +++ b/tests/fixtures/.moban-version-1234.yml @@ -1,8 +1,7 @@ moban_file_spec_version: 1234 configuration: - configuration_dir: "commons/config" + configuration_dir: "." template_dir: - - "commons/templates" - ".moban.d" configuration: data.yaml targets: diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index c6010599..4988a5de 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -441,3 +441,61 @@ def test_version_option(): from moban.main import main main() + + +@patch("logging.basicConfig") +def test_debug_option(fake_config): + fake_config.side_effect = [IOError("stop test")] + test_args = ["moban", "-d"] + with patch.object(sys, "argv", test_args): + from moban.main import main + + try: + main() + except IOError: + fake_config.assert_called_with( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + level=10, + ) + + +@patch("moban.utils.verify_the_existence_of_directories", return_value=[]) +def test_git_repo_example(_): + test_args = [ + "moban", + "-t", + "git://github.com/moremoban/pypi-mobans.git!/templates/_version.py.jj2", + "-c", + "git://github.com/moremoban/pypi-mobans.git!/config/data.yml", + "-o", + "test_git_repo_example.py", + ] + with patch.object(sys, "argv", test_args): + from moban.main import main + + main() + with open("test_git_repo_example.py") as f: + content = f.read() + eq_(content, '__version__ = "0.1.1rc3"\n__author__ = "C.W."') + os.unlink("test_git_repo_example.py") + + +@patch("moban.utils.verify_the_existence_of_directories", return_value=[]) +def test_pypi_pkg_example(_): + test_args = [ + "moban", + "-t", + "pypi://pypi-mobans-pkg/resources/templates/_version.py.jj2", + "-c", + "pypi://pypi-mobans-pkg/resources/config/data.yml", + "-o", + "test_pypi_pkg_example.py", + ] + with patch.object(sys, "argv", test_args): + from moban.main import main + + main() + with open("test_pypi_pkg_example.py") as f: + content = f.read() + eq_(content, '__version__ = "0.1.1rc3"\n__author__ = "C.W."') + os.unlink("test_pypi_pkg_example.py") diff --git a/tests/requirements.txt b/tests/requirements.txt index a6b17478..38697b96 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -12,3 +12,4 @@ pypi-mobans-pkg arrow<0.14.0;python_version=="3.4" arrow;python_version!="3.4" jinja2_time +pypifs diff --git a/tests/test_buffered_writer.py b/tests/test_buffered_writer.py new file mode 100644 index 00000000..e6684433 --- /dev/null +++ b/tests/test_buffered_writer.py @@ -0,0 +1,45 @@ +import os +import tempfile + +from moban import file_system +from nose.tools import eq_ +from moban.buffered_writer import BufferedWriter, write_file_out + +CONTENT = b""" + helloworld + + + + + """ +EXPECTED = "\n helloworld\n\n\n\n\n " + + +class TestBufferedWriter: + def setUp(self): + self.writer = BufferedWriter() + + def test_write_text(self): + test_file = "testout" + self.writer.write_file_out(test_file, CONTENT) + self.writer.close() + content = file_system.read_text(test_file) + eq_(content, EXPECTED) + os.unlink(test_file) + + def test_write_a_zip(self): + tmp_dir = os.path.normcase(tempfile.gettempdir()) + test_file = "zip://" + tmp_dir + "/testout.zip!/testout" + self.writer.write_file_out(test_file, CONTENT) + self.writer.close() + content = file_system.read_text(test_file) + eq_(content, EXPECTED) + os.unlink(os.path.join(tmp_dir, "testout.zip")) + + +def test_write_file_out(): + test_file = "testout" + write_file_out(test_file, CONTENT) + with open(test_file, "r") as f: + content = f.read() + eq_(content, EXPECTED) From 426a10dcfb81ebacc3987101eac8eb39e45a78a6 Mon Sep 17 00:00:00 2001 From: jaska Date: Sun, 18 Aug 2019 17:27:44 +0100 Subject: [PATCH 157/167] Create FUNDING.yml --- .github/FUNDING.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..23cb8a9d --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms + +patreon: chfw + From adfafe958658f822157aeebda7f3a655681851f7 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 19 Aug 2019 13:40:22 +0700 Subject: [PATCH 158/167] Tidy FUNDING.yml (#315) --- .github/FUNDING.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 23cb8a9d..7f9d74f7 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,4 +1,3 @@ # These are supported funding model platforms patreon: chfw - From 8fb867a3e42816b81f28aaf198fa6e412acceaeb Mon Sep 17 00:00:00 2001 From: jaska Date: Tue, 20 Aug 2019 07:27:05 +0100 Subject: [PATCH 159/167] Code refactoring (#316) * :fire: use released version gitfs2 for testing * :hammer: code restructuring * :hammer: split plugins folder to plugins.py and core * :tractor: relocate deprecated code to deprecated folder for easy future removal * :shirt: update coding style * :tractor: rename template to moban_factory for better readability * :shirt: update coding style * :tractor: relocate deprecated code for future removal --- moban/{copy/__init__.py => copy.py} | 0 moban/core/__init__.py | 3 + moban/{plugins => core}/context.py | 4 +- .../template.py => core/moban_factory.py} | 34 +------ moban/{plugins => core}/strategy.py | 0 moban/data_loaders/manager.py | 24 ++++- moban/deprecated/__init__.py | 95 ++++++++++++++++++- moban/{plugins => deprecated}/library.py | 0 moban/main.py | 7 +- moban/mobanfile/__init__.py | 81 ++-------------- moban/mobanfile/targets.py | 4 +- moban/{plugins/__init__.py => plugins.py} | 4 - moban/utils.py | 20 ---- rnd_requirements.txt | 2 +- .../test_command_line_options.py | 34 +++---- tests/mobanfile/test_mobanfile.py | 79 +-------------- tests/requirements.txt | 1 + tests/test_context.py | 2 +- tests/test_deprecated.py | 77 +++++++++++++++ tests/test_engine.py | 6 +- tests/test_jinja2_extensions.py | 2 +- tests/test_merge_dict.py | 2 +- tests/test_template.py | 4 +- 23 files changed, 244 insertions(+), 241 deletions(-) rename moban/{copy/__init__.py => copy.py} (100%) create mode 100644 moban/core/__init__.py rename moban/{plugins => core}/context.py (88%) rename moban/{plugins/template.py => core/moban_factory.py} (87%) rename moban/{plugins => core}/strategy.py (100%) rename moban/{plugins => deprecated}/library.py (100%) rename moban/{plugins/__init__.py => plugins.py} (81%) create mode 100644 tests/test_deprecated.py diff --git a/moban/copy/__init__.py b/moban/copy.py similarity index 100% rename from moban/copy/__init__.py rename to moban/copy.py diff --git a/moban/core/__init__.py b/moban/core/__init__.py new file mode 100644 index 00000000..ea8607c9 --- /dev/null +++ b/moban/core/__init__.py @@ -0,0 +1,3 @@ +from moban.core.moban_factory import MobanFactory + +ENGINES = MobanFactory() diff --git a/moban/plugins/context.py b/moban/core/context.py similarity index 88% rename from moban/plugins/context.py rename to moban/core/context.py index cd96570e..367ec0d6 100644 --- a/moban/plugins/context.py +++ b/moban/core/context.py @@ -1,7 +1,7 @@ import os from moban import utils, reporter, exceptions -from moban.data_loaders.manager import load_data +from moban.data_loaders.manager import merge, load_data class Context(object): @@ -15,7 +15,7 @@ def __init__(self, context_dirs): def get_data(self, file_name): try: data = load_data(self.context_dirs, file_name) - utils.merge(data, self.__cached_environ_variables) + merge(data, self.__cached_environ_variables) return data except (IOError, exceptions.IncorrectDataInput) as exception: # If data file doesn't exist: diff --git a/moban/plugins/template.py b/moban/core/moban_factory.py similarity index 87% rename from moban/plugins/template.py rename to moban/core/moban_factory.py index d1b9fae0..bb63c919 100644 --- a/moban/plugins/template.py +++ b/moban/core/moban_factory.py @@ -6,11 +6,10 @@ from fs.errors import ResourceNotFound from lml.plugin import PluginManager from moban.hashstore import HASH_STORE -from moban.deprecated import repo, deprecated +from moban.deprecated import deprecated_moban_path_notation +from moban.core.context import Context +from moban.core.strategy import Strategy from moban.buffered_writer import BufferedWriter -from moban.plugins.context import Context -from moban.plugins.library import LIBRARIES -from moban.plugins.strategy import Strategy log = logging.getLogger(__name__) PY3_ABOVE = sys.version_info[0] > 2 @@ -210,30 +209,3 @@ def expand_template_directory(directory): translated_directory = os.path.normcase(os.path.abspath(directory)) translated_directory = file_system.fs_url(translated_directory) return translated_directory - - -@deprecated(constants.MESSAGE_DEPRECATE_MOBAN_NOTATION_SINCE_0_6_0) -def deprecated_moban_path_notation(directory): - translated_directory = None - library_or_repo_name, relative_path = directory.split(":") - potential_repo_path = file_system.path_join( - repo.get_moban_home(), library_or_repo_name - ) - if file_system.exists(potential_repo_path): - # expand repo template path - if relative_path: - translated_directory = file_system.path_join( - potential_repo_path, relative_path - ) - else: - translated_directory = potential_repo_path - else: - # expand pypi template path - library_path = LIBRARIES.resource_path_of(library_or_repo_name) - if relative_path: - translated_directory = file_system.path_join( - library_path, relative_path - ) - else: - translated_directory = library_path - return translated_directory diff --git a/moban/plugins/strategy.py b/moban/core/strategy.py similarity index 100% rename from moban/plugins/strategy.py rename to moban/core/strategy.py diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index 45c518fb..77724707 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -1,6 +1,6 @@ import moban.data_loaders.yaml # noqa: F401 import moban.data_loaders.json_loader # noqa: F401 -from moban import utils, constants, file_system +from moban import constants, file_system from lml.plugin import PluginManager @@ -34,13 +34,33 @@ def load_data(base_dir, file_name): base_dir, data.pop(constants.LABEL_OVERRIDES) ) if parent_data: - return utils.merge(data, parent_data) + return merge(data, parent_data) else: return data else: return None +def merge(left, right): + """ + deep merge dictionary on the left with the one + on the right. + + Fill in left dictionary with right one where + the value of the key from the right one in + the left one is missing or None. + """ + if isinstance(left, dict) and isinstance(right, dict): + for key, value in right.items(): + if key not in left: + left[key] = value + elif left[key] is None: + left[key] = value + else: + left[key] = merge(left[key], value) + return left + + def search_file(base_dir, file_name): the_file = file_name if not file_system.exists(the_file): diff --git a/moban/deprecated/__init__.py b/moban/deprecated/__init__.py index f78f364c..a2218fbb 100644 --- a/moban/deprecated/__init__.py +++ b/moban/deprecated/__init__.py @@ -1,7 +1,18 @@ import sys from functools import wraps -from moban import reporter, constants +from moban import plugins, reporter, constants, file_system +from moban.deprecated import repo +from moban.deprecated.repo import git_clone +from moban.deprecated.library import LIBRARIES + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +KNOWN_DOMAIN_FOR_GIT = ["github.com", "gitlab.com", "bitbucket.com"] def deprecated(message): @@ -54,3 +65,85 @@ def pip_install(packages): subprocess.check_call( [sys.executable, "-m", "pip", "install", " ".join(packages)] ) + + +@deprecated(constants.MESSAGE_DEPRECATE_MOBAN_NOTATION_SINCE_0_6_0) +def deprecated_moban_path_notation(directory): + translated_directory = None + library_or_repo_name, relative_path = directory.split(":") + potential_repo_path = file_system.path_join( + repo.get_moban_home(), library_or_repo_name + ) + if file_system.exists(potential_repo_path): + # expand repo template path + if relative_path: + translated_directory = file_system.path_join( + potential_repo_path, relative_path + ) + else: + translated_directory = potential_repo_path + else: + # expand pypi template path + library_path = LIBRARIES.resource_path_of(library_or_repo_name) + if relative_path: + translated_directory = file_system.path_join( + library_path, relative_path + ) + else: + translated_directory = library_path + return translated_directory + + +@deprecated(constants.MESSAGE_DEPRECATE_REQUIRES_SINCE_0_6_0) +def handle_requires(requires): + pypi_pkgs = [] + git_repos = [] + for require in requires: + if isinstance(require, dict): + require_type = require.get(constants.REQUIRE_TYPE, "") + if require_type.upper() == constants.GIT_REQUIRE: + git_require = GitRequire( + git_url=require.get(constants.GIT_URL), + branch=require.get(constants.GIT_BRANCH), + reference=require.get(constants.GIT_REFERENCE), + submodule=require.get(constants.GIT_HAS_SUBMODULE, False), + ) + + git_repos.append(git_require) + elif require_type.upper() == constants.PYPI_REQUIRE: + pypi_pkgs.append(require.get(constants.PYPI_PACKAGE_NAME)) + else: + if is_repo(require): + git_repos.append(GitRequire(require)) + else: + pypi_pkgs.append(require) + if pypi_pkgs: + pip_install(pypi_pkgs) + plugins.make_sure_all_pkg_are_loaded() + if git_repos: + git_clone(git_repos) + + +def is_repo(require): + result = urlparse(require) + return result.scheme != "" and result.netloc in KNOWN_DOMAIN_FOR_GIT + + +@deprecated(constants.MESSAGE_DEPRECATE_COPY_SINCE_0_4_0) +def handle_copy(merged_options, copy_config): + copy_targets = [] + for (dest, src) in _iterate_list_of_dicts(copy_config): + copy_targets.append( + { + constants.LABEL_TEMPLATE: src, + constants.LABEL_OUTPUT: dest, + constants.LABEL_TEMPLATE_TYPE: constants.TEMPLATE_COPY, + } + ) + return copy_targets + + +def _iterate_list_of_dicts(list_of_dict): + for adict in list_of_dict: + for key, value in adict.items(): + yield (key, value) diff --git a/moban/plugins/library.py b/moban/deprecated/library.py similarity index 100% rename from moban/plugins/library.py rename to moban/deprecated/library.py diff --git a/moban/main.py b/moban/main.py index c5950400..ae01553c 100644 --- a/moban/main.py +++ b/moban/main.py @@ -13,11 +13,10 @@ import argparse import logging.config -from moban import plugins, reporter, constants, mobanfile, exceptions -from moban.utils import merge +from moban import core, plugins, reporter, constants, mobanfile, exceptions from moban._version import __version__ from moban.hashstore import HASH_STORE -from moban.data_loaders.manager import load_data +from moban.data_loaders.manager import merge, load_data LOG = logging.getLogger() @@ -199,7 +198,7 @@ def handle_command_line(options): act upon command options """ options = merge(options, constants.DEFAULT_OPTIONS) - engine = plugins.ENGINES.get_engine( + engine = core.ENGINES.get_engine( options[constants.LABEL_TEMPLATE_TYPE], options[constants.LABEL_TMPL_DIRS], options[constants.LABEL_CONFIG_DIR], diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index 81696ae2..4c0d1cd4 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -3,21 +3,13 @@ import sys from collections import OrderedDict -from moban import plugins, reporter, constants, file_system +from moban import core, reporter, constants, file_system from lml.utils import do_import -from moban.utils import merge, verify_the_existence_of_directories -from moban.deprecated import GitRequire, deprecated, pip_install -from moban.deprecated.repo import git_clone -from moban.plugins.template import expand_template_directories +from moban.utils import verify_the_existence_of_directories +from moban.deprecated import handle_copy, handle_requires from moban.mobanfile.targets import parse_targets, extract_group_targets - -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - - -KNOWN_DOMAIN_FOR_GIT = ["github.com", "gitlab.com", "bitbucket.com"] +from moban.core.moban_factory import expand_template_directories +from moban.data_loaders.manager import merge def find_default_moban_file(): @@ -73,11 +65,11 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): ) extensions = moban_file_configurations.get(constants.LABEL_EXTENSIONS) if extensions: - plugins.ENGINES.register_extensions(extensions) + core.ENGINES.register_extensions(extensions) template_types = merged_options.get(constants.LABEL_TEMPLATE_TYPES) if template_types: - plugins.ENGINES.register_options(template_types) + core.ENGINES.register_options(template_types) if cli_target: number_of_templated_files = handle_targets( @@ -93,26 +85,6 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): return exit_code -@deprecated(constants.MESSAGE_DEPRECATE_COPY_SINCE_0_4_0) -def handle_copy(merged_options, copy_config): - copy_targets = [] - for (dest, src) in _iterate_list_of_dicts(copy_config): - copy_targets.append( - { - constants.LABEL_TEMPLATE: src, - constants.LABEL_OUTPUT: dest, - constants.LABEL_TEMPLATE_TYPE: constants.TEMPLATE_COPY, - } - ) - return copy_targets - - -def _iterate_list_of_dicts(list_of_dict): - for adict in list_of_dict: - for key, value in adict.items(): - yield (key, value) - - def handle_targets(merged_options, targets): list_of_templating_parameters = parse_targets(merged_options, targets) jobs_for_each_engine = OrderedDict() @@ -125,7 +97,7 @@ def handle_targets(merged_options, targets): target.set_template_type(forced_template_type) template_type = target.template_type - primary_template_type = plugins.ENGINES.get_primary_key(template_type) + primary_template_type = core.ENGINES.get_primary_key(template_type) if primary_template_type is None: primary_template_type = merged_options[ constants.LABEL_TEMPLATE_TYPE @@ -139,7 +111,7 @@ def handle_targets(merged_options, targets): count = 0 for template_type in jobs_for_each_engine.keys(): - engine = plugins.ENGINES.get_engine( + engine = core.ENGINES.get_engine( template_type, merged_options[constants.LABEL_TMPL_DIRS], merged_options[constants.LABEL_CONFIG_DIR], @@ -185,38 +157,3 @@ def extract_target(options): else: result = {output: template} return result - - -@deprecated(constants.MESSAGE_DEPRECATE_REQUIRES_SINCE_0_6_0) -def handle_requires(requires): - pypi_pkgs = [] - git_repos = [] - for require in requires: - if isinstance(require, dict): - require_type = require.get(constants.REQUIRE_TYPE, "") - if require_type.upper() == constants.GIT_REQUIRE: - git_require = GitRequire( - git_url=require.get(constants.GIT_URL), - branch=require.get(constants.GIT_BRANCH), - reference=require.get(constants.GIT_REFERENCE), - submodule=require.get(constants.GIT_HAS_SUBMODULE, False), - ) - - git_repos.append(git_require) - elif require_type.upper() == constants.PYPI_REQUIRE: - pypi_pkgs.append(require.get(constants.PYPI_PACKAGE_NAME)) - else: - if is_repo(require): - git_repos.append(GitRequire(require)) - else: - pypi_pkgs.append(require) - if pypi_pkgs: - pip_install(pypi_pkgs) - plugins.make_sure_all_pkg_are_loaded() - if git_repos: - git_clone(git_repos) - - -def is_repo(require): - result = urlparse(require) - return result.scheme != "" and result.netloc in KNOWN_DOMAIN_FOR_GIT diff --git a/moban/mobanfile/targets.py b/moban/mobanfile/targets.py index e32c8767..2ec64a5d 100644 --- a/moban/mobanfile/targets.py +++ b/moban/mobanfile/targets.py @@ -1,6 +1,6 @@ import uuid -from moban import plugins, reporter, constants, exceptions +from moban import core, reporter, constants, exceptions from moban.definitions import TemplateTarget from moban.mobanfile.templates import handle_template @@ -65,7 +65,7 @@ def _handle_explicit_target(options, target): constants.TEMPLATE_TYPES_OPTIONS: template_types_options, } } - plugins.ENGINES.register_options(the_adhoc_type) + core.ENGINES.register_options(the_adhoc_type) template_type = file_extension for src, dest, t_type in handle_template( template_file, output, options[constants.LABEL_TMPL_DIRS] diff --git a/moban/plugins/__init__.py b/moban/plugins.py similarity index 81% rename from moban/plugins/__init__.py rename to moban/plugins.py index 5cfdf27e..88ce7f65 100644 --- a/moban/plugins/__init__.py +++ b/moban/plugins.py @@ -1,6 +1,5 @@ from moban import constants from lml.loader import scan_plugins_regex -from moban.plugins.template import MobanFactory BUILTIN_EXENSIONS = [ "moban.jinja2.engine", @@ -10,8 +9,5 @@ ] -ENGINES = MobanFactory() - - def make_sure_all_pkg_are_loaded(): scan_plugins_regex(constants.MOBAN_ALL, "moban", None, BUILTIN_EXENSIONS) diff --git a/moban/utils.py b/moban/utils.py index 23e53f62..d4e4aa11 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -9,26 +9,6 @@ PY2 = sys.version_info[0] == 2 -def merge(left, right): - """ - deep merge dictionary on the left with the one - on the right. - - Fill in left dictionary with right one where - the value of the key from the right one in - the left one is missing or None. - """ - if isinstance(left, dict) and isinstance(right, dict): - for key, value in right.items(): - if key not in left: - left[key] = value - elif left[key] is None: - left[key] = value - else: - left[key] = merge(left[key], value) - return left - - def mkdir_p(path): try: os.makedirs(path) diff --git a/rnd_requirements.txt b/rnd_requirements.txt index baf71891..79f046bb 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,5 +1,5 @@ https://github.com/moremoban/pyfilesystem2/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip -https://github.com/moremoban/gitfs2/archive/master.zip + diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index 4988a5de..bc732cca 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -18,7 +18,7 @@ def setUp(self): self.patcher1.start() @patch("moban.file_system.abspath") - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_custom_options(self, fake_template_doer, fake_abspath): test_args = [ "moban", @@ -39,7 +39,7 @@ def test_custom_options(self, fake_template_doer, fake_abspath): "a.jj2", "config.yaml", "moban.output" ) - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_minimal_options(self, fake_template_doer): test_args = ["moban", "-c", self.config_file, "-t", "a.jj2"] with patch.object(sys, "argv", test_args): @@ -73,7 +73,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_default_options(self, fake_template_doer): test_args = ["moban", "-t", "a.jj2"] with patch.object(sys, "argv", test_args): @@ -84,7 +84,7 @@ def test_default_options(self, fake_template_doer): "a.jj2", "data.yml", "moban.output" ) - @patch("moban.plugins.template.MobanEngine.render_string_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_string_to_file") def test_string_template(self, fake_template_doer): string_template = "{{HELLO}}" test_args = ["moban", string_template] @@ -133,7 +133,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command(self, fake_template_doer): test_args = ["moban"] with patch.object(sys, "argv", test_args): @@ -152,7 +152,7 @@ def test_single_command(self, fake_template_doer): ) @raises(Exception) - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command_with_missing_output(self, fake_template_doer): test_args = ["moban", "-t", "README.rst.jj2"] with patch.object(sys, "argv", test_args): @@ -160,7 +160,7 @@ def test_single_command_with_missing_output(self, fake_template_doer): main() - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command_with_a_few_options(self, fake_template_doer): test_args = ["moban", "-t", "README.rst.jj2", "-o", "xyz.output"] with patch.object(sys, "argv", test_args): @@ -174,7 +174,7 @@ def test_single_command_with_a_few_options(self, fake_template_doer): [TemplateTarget("README.rst.jj2", "data.yaml", "xyz.output")], ) - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command_with_options(self, fake_template_doer): test_args = [ "moban", @@ -224,7 +224,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command(self, fake_template_doer): test_args = ["moban"] with patch.object(sys, "argv", test_args): @@ -262,7 +262,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command(self, fake_template_doer): test_args = ["moban", "-m", self.config_file] with patch.object(sys, "argv", test_args): @@ -297,7 +297,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_template_option_override_moban_file(self, fake_template_doer): test_args = ["moban", "-t", "setup.py.jj2"] with patch.object(sys, "argv", test_args): @@ -308,7 +308,7 @@ def test_template_option_override_moban_file(self, fake_template_doer): "setup.py.jj2", "data.yml", "moban.output" ) - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_template_option_not_in_moban_file(self, fake_template_doer): test_args = ["moban", "-t", "foo.jj2"] with patch.object(sys, "argv", test_args): @@ -341,7 +341,7 @@ def setUp(self): self.config_file = ".moban.yml" @raises(SystemExit) - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_no_configuration(self, fake_template_doer): with open(self.config_file, "w") as f: f.write("") @@ -352,7 +352,7 @@ def test_no_configuration(self, fake_template_doer): main() @raises(SystemExit) - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_no_configuration_2(self, fake_template_doer): with open(self.config_file, "w") as f: f.write("not: related") @@ -363,7 +363,7 @@ def test_no_configuration_2(self, fake_template_doer): main() @raises(SystemExit) - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_no_targets(self, fake_template_doer): with open(self.config_file, "w") as f: f.write("configuration: test") @@ -394,7 +394,7 @@ def test_single_command(self, _): from moban.main import main with patch( - "moban.plugins.template.MobanEngine.render_to_files" + "moban.core.moban_factory.MobanEngine.render_to_files" ) as fake: main() call_args = list(fake.call_args[0][0]) @@ -419,7 +419,7 @@ def setUp(self): with open(self.config_file, "w") as f: f.write("hello: world") - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_mako_option(self, fake_template_doer): test_args = ["moban", "-t", "a.mako"] with patch.object(sys, "argv", test_args): diff --git a/tests/mobanfile/test_mobanfile.py b/tests/mobanfile/test_mobanfile.py index f4cb6487..49552c2e 100644 --- a/tests/mobanfile/test_mobanfile.py +++ b/tests/mobanfile/test_mobanfile.py @@ -1,7 +1,6 @@ import fs.path from mock import patch from nose.tools import eq_ -from moban.deprecated import GitRequire from moban.definitions import TemplateTarget @@ -37,81 +36,7 @@ def test_no_moban_file(self): assert actual is None -@patch("moban.mobanfile.pip_install") -def test_handle_requires_pypkg(fake_pip_install): - modules = ["package1", "package2"] - from moban.mobanfile import handle_requires - - handle_requires(modules) - fake_pip_install.assert_called_with(modules) - - -@patch("moban.mobanfile.pip_install") -def test_handle_requires_pypkg_with_alternative_syntax(fake_pip_install): - modules = [{"type": "pypi", "name": "pypi-mobans"}] - from moban.mobanfile import handle_requires - - handle_requires(modules) - fake_pip_install.assert_called_with(["pypi-mobans"]) - - -@patch("moban.mobanfile.git_clone") -def test_handle_requires_repos(fake_git_clone): - repos = ["https://github.com/my/repo", "https://gitlab.com/my/repo"] - from moban.mobanfile import handle_requires - - expected = [] - for repo in repos: - expected.append(GitRequire(git_url=repo, submodule=False)) - - handle_requires(repos) - fake_git_clone.assert_called_with(expected) - - -@patch("moban.mobanfile.git_clone") -def test_handle_requires_repos_with_alternative_syntax(fake_git_clone): - repos = [{"type": "git", "url": "https://github.com/my/repo"}] - from moban.mobanfile import handle_requires - - handle_requires(repos) - fake_git_clone.assert_called_with( - [GitRequire(git_url="https://github.com/my/repo")] - ) - - -@patch("moban.mobanfile.pip_install") -@patch("moban.mobanfile.git_clone") -def test_handle_requires_repos_with_submodule( - fake_git_clone, fake_pip_install -): - repos = [ - {"type": "git", "url": "https://github.com/my/repo", "submodule": True} - ] - from moban.mobanfile import handle_requires - - handle_requires(repos) - fake_git_clone.assert_called_with( - [GitRequire(git_url="https://github.com/my/repo", submodule=True)] - ) - eq_(fake_pip_install.called, False) - - -def test_is_repo(): - repos = [ - "https://github.com/my/repo", - "https://gitlab.com/my/repo", - "https://bitbucket.com/my/repo", - "https://unsupported.com/my/repo", - "invalid/repo/url", - ] - from moban.mobanfile import is_repo - - actual = [is_repo(repo) for repo in repos] - expected = [True, True, True, False, False] - eq_(expected, actual) - - -@patch("moban.plugins.template.MobanEngine.render_to_files") +@patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_handle_targets(fake_renderer): from moban.mobanfile import handle_targets @@ -144,7 +69,7 @@ def test_handle_targets(fake_renderer): ) -@patch("moban.plugins.template.MobanEngine.render_to_files") +@patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_handle_targets_sequence(fake_renderer): from moban.mobanfile import handle_targets diff --git a/tests/requirements.txt b/tests/requirements.txt index 38697b96..055d0047 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -13,3 +13,4 @@ arrow<0.14.0;python_version=="3.4" arrow;python_version!="3.4" jinja2_time pypifs +gitfs2 diff --git a/tests/test_context.py b/tests/test_context.py index 72575582..846d2383 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -2,7 +2,7 @@ import fs.path from nose.tools import eq_ -from moban.plugins.context import Context +from moban.core.context import Context def test_context(): diff --git a/tests/test_deprecated.py b/tests/test_deprecated.py new file mode 100644 index 00000000..fa427c8f --- /dev/null +++ b/tests/test_deprecated.py @@ -0,0 +1,77 @@ +from mock import patch +from nose.tools import eq_ +from moban.deprecated import GitRequire + + +@patch("moban.deprecated.pip_install") +def test_handle_requires_pypkg(fake_pip_install): + modules = ["package1", "package2"] + from moban.deprecated import handle_requires + + handle_requires(modules) + fake_pip_install.assert_called_with(modules) + + +@patch("moban.deprecated.pip_install") +def test_handle_requires_pypkg_with_alternative_syntax(fake_pip_install): + modules = [{"type": "pypi", "name": "pypi-mobans"}] + from moban.mobanfile import handle_requires + + handle_requires(modules) + fake_pip_install.assert_called_with(["pypi-mobans"]) + + +@patch("moban.deprecated.git_clone") +def test_handle_requires_repos(fake_git_clone): + repos = ["https://github.com/my/repo", "https://gitlab.com/my/repo"] + from moban.mobanfile import handle_requires + + expected = [] + for repo in repos: + expected.append(GitRequire(git_url=repo, submodule=False)) + + handle_requires(repos) + fake_git_clone.assert_called_with(expected) + + +@patch("moban.deprecated.git_clone") +def test_handle_requires_repos_with_alternative_syntax(fake_git_clone): + repos = [{"type": "git", "url": "https://github.com/my/repo"}] + from moban.mobanfile import handle_requires + + handle_requires(repos) + fake_git_clone.assert_called_with( + [GitRequire(git_url="https://github.com/my/repo")] + ) + + +@patch("moban.deprecated.pip_install") +@patch("moban.deprecated.git_clone") +def test_handle_requires_repos_with_submodule( + fake_git_clone, fake_pip_install +): + repos = [ + {"type": "git", "url": "https://github.com/my/repo", "submodule": True} + ] + from moban.mobanfile import handle_requires + + handle_requires(repos) + fake_git_clone.assert_called_with( + [GitRequire(git_url="https://github.com/my/repo", submodule=True)] + ) + eq_(fake_pip_install.called, False) + + +def test_is_repo(): + repos = [ + "https://github.com/my/repo", + "https://gitlab.com/my/repo", + "https://bitbucket.com/my/repo", + "https://unsupported.com/my/repo", + "invalid/repo/url", + ] + from moban.deprecated import is_repo + + actual = [is_repo(repo) for repo in repos] + expected = [True, True, True, False, False] + eq_(expected, actual) diff --git a/tests/test_engine.py b/tests/test_engine.py index 1739db21..c5f4b179 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -5,15 +5,15 @@ import moban.exceptions as exceptions from mock import patch from lml.plugin import PluginInfo +from moban.core import ENGINES from nose.tools import eq_, raises -from moban.plugins import ENGINES +from moban.core.context import Context from moban.jinja2.engine import ( Engine, is_extension_list_valid, import_module_of_extension, ) -from moban.plugins.context import Context -from moban.plugins.template import MobanEngine, expand_template_directories +from moban.core.moban_factory import MobanEngine, expand_template_directories USER_HOME = fs.path.join("user", "home", ".moban", "repos") diff --git a/tests/test_jinja2_extensions.py b/tests/test_jinja2_extensions.py index 48ddbdb9..61ddbf1a 100644 --- a/tests/test_jinja2_extensions.py +++ b/tests/test_jinja2_extensions.py @@ -3,8 +3,8 @@ from moban import file_system from nose.tools import eq_ from moban.jinja2.engine import Engine -from moban.plugins.template import MobanEngine from moban.jinja2.extensions import jinja_global +from moban.core.moban_factory import MobanEngine def test_globals(): diff --git a/tests/test_merge_dict.py b/tests/test_merge_dict.py index 55cc01d2..445f62db 100644 --- a/tests/test_merge_dict.py +++ b/tests/test_merge_dict.py @@ -1,4 +1,4 @@ -from moban.utils import merge +from moban.data_loaders.manager import merge def test_simple_union(): diff --git a/tests/test_template.py b/tests/test_template.py index 091f9f0c..d1c6c790 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -2,13 +2,13 @@ import fs.path from mock import patch +from moban.core import ENGINES from nose.tools import eq_ -from moban.plugins import ENGINES from moban.definitions import TemplateTarget from moban.jinja2.engine import Engine from moban.data_loaders.yaml import open_yaml -MODULE = "moban.plugins.template" +MODULE = "moban.core.moban_factory" @patch(MODULE + ".MobanEngine._render_with_finding_data_first") From e5c26b012023f29f25722135fe5a55b547c018c6 Mon Sep 17 00:00:00 2001 From: jaska Date: Wed, 21 Aug 2019 08:44:21 +0100 Subject: [PATCH 160/167] :hammer: :tractor: refactored tests (#317) * :hammer: :tractor: refactored tests * :books: enhanced PR template * :shirt: :tractor: beautify the test codes * :hammer: code refactoring on test docs * :hammer: fully transform the docs based tests * :microscope: test assertions for level 21 * :shirt: coding style update * :fire: deduplication --- .github/PULL_REQUEST_TEMPLATE.md | 15 +- tests/{ => core}/test_context.py | 0 .../{test_template.py => core/test_engine.py} | 0 .../test_moban_factory.py} | 0 tests/{ => data_loaders}/test_json_loader.py | 0 tests/{ => data_loaders}/test_merge_dict.py | 0 tests/{ => data_loaders}/test_yaml_loader.py | 0 .../test_handle_requires.py} | 0 .../test_engine.py} | 0 .../test_extensions.py} | 0 .../test_github.py} | 0 .../test_repr.py} | 0 .../test_text.py} | 0 tests/test_copy_encoding.py | 21 -- tests/test_copy_engine.py | 18 ++ tests/test_docs.py | 223 +++++++----------- tests/test_regression.py | 68 ++---- tests/utils.py | 64 +++++ 18 files changed, 191 insertions(+), 218 deletions(-) rename tests/{ => core}/test_context.py (100%) rename tests/{test_template.py => core/test_engine.py} (100%) rename tests/{test_engine.py => core/test_moban_factory.py} (100%) rename tests/{ => data_loaders}/test_json_loader.py (100%) rename tests/{ => data_loaders}/test_merge_dict.py (100%) rename tests/{ => data_loaders}/test_yaml_loader.py (100%) rename tests/{test_deprecated.py => deprecated/test_handle_requires.py} (100%) rename tests/{test_jinja2_engine.py => jinja2/test_engine.py} (100%) rename tests/{test_jinja2_extensions.py => jinja2/test_extensions.py} (100%) rename tests/{test_filter_github.py => jinja2/test_github.py} (100%) rename tests/{test_filter_repr.py => jinja2/test_repr.py} (100%) rename tests/{test_text_filter.py => jinja2/test_text.py} (100%) delete mode 100644 tests/test_copy_encoding.py create mode 100644 tests/utils.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d29a3ddb..89283248 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,8 @@ Before raising the PR, here is a check list: -1) have you written unit tests for your code changes? -2) have you run "make format"? -3) are you requesting to "dev"? -4) have you updated the change log? -5) do you think that you can understand your changes after 6 month? - 5.1) can someone else understand your changes without your explanation? -6) are you pround of your code changes? - 6.1) do you have the feeling of achievement? -7) please add your name and github link to contributors.rst in alphabetical order. \ No newline at end of file +- [ ] have you written unit tests for your code changes? +- [ ] have you updated the change log? +- [ ] can someone else understand your changes without your explanation? +- [ ] are you pround of your code changes? +- [ ] please add your name and github link to contributors.rst in alphabetical order. + diff --git a/tests/test_context.py b/tests/core/test_context.py similarity index 100% rename from tests/test_context.py rename to tests/core/test_context.py diff --git a/tests/test_template.py b/tests/core/test_engine.py similarity index 100% rename from tests/test_template.py rename to tests/core/test_engine.py diff --git a/tests/test_engine.py b/tests/core/test_moban_factory.py similarity index 100% rename from tests/test_engine.py rename to tests/core/test_moban_factory.py diff --git a/tests/test_json_loader.py b/tests/data_loaders/test_json_loader.py similarity index 100% rename from tests/test_json_loader.py rename to tests/data_loaders/test_json_loader.py diff --git a/tests/test_merge_dict.py b/tests/data_loaders/test_merge_dict.py similarity index 100% rename from tests/test_merge_dict.py rename to tests/data_loaders/test_merge_dict.py diff --git a/tests/test_yaml_loader.py b/tests/data_loaders/test_yaml_loader.py similarity index 100% rename from tests/test_yaml_loader.py rename to tests/data_loaders/test_yaml_loader.py diff --git a/tests/test_deprecated.py b/tests/deprecated/test_handle_requires.py similarity index 100% rename from tests/test_deprecated.py rename to tests/deprecated/test_handle_requires.py diff --git a/tests/test_jinja2_engine.py b/tests/jinja2/test_engine.py similarity index 100% rename from tests/test_jinja2_engine.py rename to tests/jinja2/test_engine.py diff --git a/tests/test_jinja2_extensions.py b/tests/jinja2/test_extensions.py similarity index 100% rename from tests/test_jinja2_extensions.py rename to tests/jinja2/test_extensions.py diff --git a/tests/test_filter_github.py b/tests/jinja2/test_github.py similarity index 100% rename from tests/test_filter_github.py rename to tests/jinja2/test_github.py diff --git a/tests/test_filter_repr.py b/tests/jinja2/test_repr.py similarity index 100% rename from tests/test_filter_repr.py rename to tests/jinja2/test_repr.py diff --git a/tests/test_text_filter.py b/tests/jinja2/test_text.py similarity index 100% rename from tests/test_text_filter.py rename to tests/jinja2/test_text.py diff --git a/tests/test_copy_encoding.py b/tests/test_copy_encoding.py deleted file mode 100644 index b388f8c7..00000000 --- a/tests/test_copy_encoding.py +++ /dev/null @@ -1,21 +0,0 @@ -import fs.path -from moban import file_system -from moban.copy import ContentForwardEngine -from nose.tools import eq_ - - -class TestCopyEncoding: - def setUp(self): - template_path = fs.path.join("tests", "fixtures") - template_fs = file_system.get_multi_fs([template_path]) - self.engine = ContentForwardEngine(template_fs) - - def test_encoding_of_template(self): - template_content = self.engine.get_template("coala_color.svg") - with open("tests/fixtures/coala_color.svg", "rb") as expected: - expected = expected.read() - eq_(expected, template_content) - template_content = self.engine.get_template("non-unicode.char") - with open("tests/fixtures/non-unicode.char", "rb") as expected: - expected = expected.read() - eq_(expected, template_content) diff --git a/tests/test_copy_engine.py b/tests/test_copy_engine.py index fae1c5f1..65a25262 100644 --- a/tests/test_copy_engine.py +++ b/tests/test_copy_engine.py @@ -1,5 +1,6 @@ import os +import fs.path from moban import file_system from moban.copy import ContentForwardEngine from nose.tools import eq_ @@ -31,3 +32,20 @@ def test_apply_template(self): test_content = "simply forwarded" template_content = self.engine.apply_template(test_content, "not used") eq_(test_content, template_content) + + +class TestCopyEncoding: + def setUp(self): + template_path = fs.path.join("tests", "fixtures") + template_fs = file_system.get_multi_fs([template_path]) + self.engine = ContentForwardEngine(template_fs) + + def test_encoding_of_template(self): + template_content = self.engine.get_template("coala_color.svg") + with open("tests/fixtures/coala_color.svg", "rb") as expected: + expected = expected.read() + eq_(expected, template_content) + template_content = self.engine.get_template("non-unicode.char") + with open("tests/fixtures/non-unicode.char", "rb") as expected: + expected = expected.read() + eq_(expected, template_content) diff --git a/tests/test_docs.py b/tests/test_docs.py index 377a4dbd..254ce95b 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,46 +1,30 @@ import os -import sys -from textwrap import dedent -from mock import patch -from moban import file_system -from moban.main import main from nose.tools import eq_ -from fs.opener.parse import parse_fs_url +from .utils import Docs, custom_dedent -def custom_dedent(long_texts): - refined = dedent(long_texts) - if refined.startswith("\n"): - refined = refined[1:] - return refined - - -class TestTutorial: - def setUp(self): - self.current = os.getcwd() +class TestTutorial(Docs): def test_level_1(self): expected = "world" folder = "level-1-jinja2-cli" self._moban(folder, expected) def test_level_2(self): - expected = custom_dedent( - """ + expected = """ ========header============ world ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-2-template-inheritance" self._moban(folder, expected) def test_level_3(self): - expected = custom_dedent( - """ + expected = """ ========header============ world @@ -49,13 +33,12 @@ def test_level_3(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-3-data-override" self._moban(folder, expected) def test_level_4(self): - expected = custom_dedent( - """ + expected = """ ========header============ world @@ -64,13 +47,12 @@ def test_level_4(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-4-single-command" - self._raw_moban(["moban"], folder, expected, "a.output") + self.run_moban(["moban"], folder, [("a.output", expected)]) def test_level_5(self): - expected = custom_dedent( - """ + expected = """ ========header============ world @@ -81,13 +63,12 @@ def test_level_5(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-5-custom-configuration" - self._raw_moban(["moban"], folder, expected, "a.output") + self.run_moban(["moban"], folder, [("a.output", expected)]) def test_level_6(self): - expected = custom_dedent( - """ + expected = """ ========header============ world2 @@ -98,13 +79,12 @@ def test_level_6(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-6-complex-configuration" - self._raw_moban(["moban"], folder, expected, "a.output2") + self.run_moban(["moban"], folder, [("a.output2", expected)]) def test_level_20(self): - expected = custom_dedent( - """ + expected = """ ========header============ world2 @@ -115,80 +95,77 @@ def test_level_20(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-20-templates-configs-in-zip-or-tar" - self._raw_moban_with_fs( - ["moban"], folder, expected, "zip://a.zip!/a.output2" + self.run_moban_with_fs( + ["moban"], folder, [("zip://a.zip!/a.output2", expected)] ) def test_level_7(self): - expected = custom_dedent( - """ + expected = """ Hello, you are in level 7 example Hello, you are not in level 7 """ - ) + expected = custom_dedent(expected) + folder = "level-7-use-custom-jinja2-filter-test-n-global" - self._raw_moban(["moban"], folder, expected, "test.output") + self.run_moban(["moban"], folder, [("test.output", expected)]) def test_level_8(self): expected = "it is a test\n" folder = "level-8-pass-a-folder-full-of-templates" check_file = os.path.join("templated-folder", "my") - self._raw_moban(["moban"], folder, expected, check_file) + self.run_moban(["moban"], folder, [(check_file, expected)]) def test_level_9(self): expected = "pypi-mobans: moban dependency as pypi package" folder = "level-9-moban-dependency-as-pypi-package" - self._raw_moban(["moban"], folder, expected, "test.txt") + self.run_moban(["moban"], folder, [("test.txt", expected)]) def test_level_9_deprecated(self): expected = "pypi-mobans: moban dependency as pypi package" folder = "deprecated-level-9-moban-dependency-as-pypi-package" - self._raw_moban(["moban"], folder, expected, "test.txt") + self.run_moban(["moban"], folder, [("test.txt", expected)]) def test_level_10(self): expected = "pypi-mobans: moban dependency as git repo" folder = "level-10-moban-dependency-as-git-repo" - self._raw_moban(["moban"], folder, expected, "test.txt") + self.run_moban(["moban"], folder, [("test.txt", expected)]) def test_level_10_deprecated(self): expected = "pypi-mobans: moban dependency as git repo" folder = "deprecated-level-10-moban-dependency-as-git-repo" - self._raw_moban(["moban"], folder, expected, "test.txt") + self.run_moban(["moban"], folder, [("test.txt", expected)]) def test_level_11(self): expected = "handlebars does not support inheritance\n" folder = "level-11-use-handlebars" - self._raw_moban(["moban"], folder, expected, "a.output") + self.run_moban(["moban"], folder, [("a.output", expected)]) - def test_level_12_a(self): - expected_a = custom_dedent( - """ + def test_level_12(self): + expected_a = """ world world world world """ - ) - folder = "level-12-use-template-engine-extensions" - self._raw_moban(["moban"], folder, expected_a, "a.output") - - def test_level_12_b(self): - expected_b = custom_dedent( - """ + expected_b = """ 142 42 142 """ - ) + expected_a = custom_dedent(expected_a) + expected_b = custom_dedent(expected_b) folder = "level-12-use-template-engine-extensions" - self._raw_moban(["moban"], folder, expected_b, "b.output") + self.run_moban( + ["moban"], + folder, + [("a.output", expected_a), ("b.output", expected_b)], + ) def test_level_13_json(self): - expected = custom_dedent( - """ + expected = """ ========header============ world from child.json @@ -197,14 +174,13 @@ def test_level_13_json(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-13-any-data-override-any-data" commands = ["moban", "-c", "child.json", "-t", "a.template"] - self._raw_moban(commands, folder, expected, "moban.output") + self.run_moban(commands, folder, [("moban.output", expected)]) def test_level_13_yaml(self): - expected = custom_dedent( - """ + expected = """ ========header============ world from child.yaml @@ -213,14 +189,13 @@ def test_level_13_yaml(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-13-any-data-override-any-data" commands = ["moban", "-c", "child.yaml", "-t", "a.template"] - self._raw_moban(commands, folder, expected, "moban.output") + self.run_moban(commands, folder, [("moban.output", expected)]) def test_level_14_custom(self): - expected = custom_dedent( - """ + expected = """ ========header============ world from child.cusom @@ -229,34 +204,40 @@ def test_level_14_custom(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-14-custom-data-loader" commands = ["moban"] - self._raw_moban(commands, folder, expected, "a.output") + self.run_moban(commands, folder, [("a.output", expected)]) def test_level_15_copy_templates_as_target(self): expected = "test file\n" folder = "level-15-copy-templates-as-target" - self._raw_moban(["moban"], folder, expected, "simple.file") - - _verify_content( - "target_without_template_type", - "file extension will trigger copy engine\n", - ) - _verify_content( - "target_in_short_form", + assertions = [ + ("simple.file", expected), ( - "it is OK to have a short form, " - + "but the file to be 'copied' shall have 'copy' extension, " - + "so as to trigger ContentForwardEngine, 'copy' engine.\n" + "target_without_template_type", + "file extension will trigger copy engine\n", ), - ) + ( + "target_in_short_form", + ( + "it is OK to have a short form, " + + "but the file to be 'copied' shall have 'copy' extension, " + + "so as to trigger ContentForwardEngine, 'copy' engine.\n" + ), + ), + ] + self.run_moban(["moban"], folder, assertions) def test_level_21_copy_templates_into_zips(self): expected = "test file\n" folder = "level-21-copy-templates-into-an-alien-file-system" + long_url = ( + "zip://my.zip!/test-recursive-dir/sub_directory_is_copied" + + "/because_star_star_is_specified.txt" + ) criterias = [ ["zip://my.zip!/simple.file", expected], [ @@ -271,20 +252,22 @@ def test_level_21_copy_templates_into_zips(self): + "so as to trigger ContentForwardEngine, 'copy' engine.\n" ), ], + ["zip://my.zip!/test-dir/afile.txt", "dir for copying\n"], + [long_url, "dest_directory: source_directory/**\n"], ] - self._raw_moban_with_fs2(["moban"], folder, criterias) + self.run_moban_with_fs(["moban"], folder, criterias) def test_level_16_group_targets_using_template_type(self): expected = "test file\n" folder = "level-16-group-targets-using-template-type" - self._raw_moban(["moban"], folder, expected, "simple.file") + self.run_moban(["moban"], folder, [("simple.file", expected)]) def test_level_17_force_template_type_from_moban_file(self): expected = "test file\n" folder = "level-17-force-template-type-from-moban-file" - self._raw_moban(["moban"], folder, expected, "simple.file") + self.run_moban(["moban"], folder, [("simple.file", expected)]) def test_level_18_user_defined_template_types(self): from datetime import datetime @@ -292,24 +275,28 @@ def test_level_18_user_defined_template_types(self): expected = "{date}\n".format(date=datetime.now().strftime("%Y-%m-%d")) folder = "level-18-user-defined-template-types" - self._raw_moban(["moban"], folder, expected, "a.output") - - _verify_content("b.output", "shijie\n") + self.run_moban( + ["moban"], + folder, + [("a.output", expected), ("b.output", "shijie\n")], + ) def test_level_19_without_group_target(self): expected = "test file\n" folder = "level-19-moban-a-sub-group-in-targets" - self._raw_moban(["moban"], folder, expected, "simple.file") - _verify_content("a.output", "I will not be selected in level 19\n") - os.unlink("a.output") + assertions = [ + ("simple.file", expected), + ("a.output", "I will not be selected in level 19\n"), + ] + self.run_moban(["moban"], folder, assertions) def test_level_19_with_group_target(self): expected = "test file\n" folder = "level-19-moban-a-sub-group-in-targets" - self._raw_moban( - ["moban", "-g", "copy"], folder, expected, "simple.file" + self.run_moban( + ["moban", "-g", "copy"], folder, [("simple.file", expected)] ) # make sure only copy target is executed eq_(False, os.path.exists("a.output")) @@ -318,48 +305,8 @@ def test_misc_1(self): expected = "test file\n" folder = "misc-1-copying-templates" - self._raw_moban(["moban"], folder, expected, "simple.file") + self.run_moban(["moban"], folder, [("simple.file", expected)]) def _moban(self, folder, expected): args = ["moban", "-c", "data.yml", "-t", "a.template"] - self._raw_moban(args, folder, expected, "moban.output") - - def _raw_moban(self, args, folder, expected, output): - os.chdir(os.path.join("docs", folder)) - with patch.object(sys, "argv", args): - main() - _verify_content(output, expected) - os.unlink(output) - - def _raw_moban_with_fs(self, args, folder, expected, output): - os.chdir(os.path.join("docs", folder)) - with patch.object(sys, "argv", args): - main() - _verify_content_with_fs(output, expected) - result = parse_fs_url(output) - os.unlink(result.resource) # delete the zip file - - def _raw_moban_with_fs2(self, args, folder, criterias): - os.chdir(os.path.join("docs", folder)) - with patch.object(sys, "argv", args): - main() - - for output, expected in criterias: - _verify_content_with_fs(output, expected) - result = parse_fs_url(output) - os.unlink(result.resource) # delete the zip file - - def tearDown(self): - os.unlink(".moban.hashes") - os.chdir(self.current) - - -def _verify_content(file_name, expected): - with open(file_name, "r") as f: - content = f.read() - eq_(content, expected) - - -def _verify_content_with_fs(file_name, expected): - content = file_system.read_unicode(file_name) - eq_(content, expected) + self.run_moban(args, folder, [("moban.output", expected)]) diff --git a/tests/test_regression.py b/tests/test_regression.py index 1136669f..85ab9a48 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -1,26 +1,18 @@ import os import sys import filecmp -from textwrap import dedent import fs.path from mock import patch -from moban import file_system from moban.main import main -from nose.tools import eq_ -from fs.opener.parse import parse_fs_url +from .utils import Docs -def custom_dedent(long_texts): - refined = dedent(long_texts) - if refined.startswith("\n"): - refined = refined[1:] - return refined - -class TestRegression: +class TestRegression(Docs): def setUp(self): - self.current = os.getcwd() + super(TestRegression, self).setUp() + self.base_folder = fs.path.join("tests", "regression_tests") def test_coping_binary_file(self): folder = "regr-01-copy-binary-file" @@ -36,6 +28,10 @@ def test_level_21_copy_templates_into_tars(self): expected = "test file\n" folder = "level-21-b-copy-templates-into-a-tar" + long_url = ( + "tar://my.tar!/test-recursive-dir/sub_directory_is_copied" + + "/because_star_star_is_specified.txt" + ) criterias = [ ["tar://my.tar!/simple.file", expected], [ @@ -50,13 +46,19 @@ def test_level_21_copy_templates_into_tars(self): + "so as to trigger ContentForwardEngine, 'copy' engine.\n" ), ], + ["tar://my.tar!/test-dir/afile.txt", "dir for copying\n"], + [long_url, "dest_directory: source_directory/**\n"], ] - self._raw_moban_with_fs2(["moban"], folder, criterias) + self.run_moban_with_fs(["moban"], folder, criterias) def test_level_21_copy_templates_from_tars(self): expected = "test file\n" folder = "level-21-c-copy-templates-from-a-tar" + long_url = ( + "zip://my.zip!/test-recursive-dir/sub_directory_is_copied" + + "/because_star_star_is_specified.txt" + ) criterias = [ ["zip://my.zip!/simple.file", expected], [ @@ -71,8 +73,10 @@ def test_level_21_copy_templates_from_tars(self): + "so as to trigger ContentForwardEngine, 'copy' engine.\n" ), ], + ["zip://my.zip!/test-dir/afile.txt", "dir for copying\n"], + [long_url, "dest_directory: source_directory/**\n"], ] - self._raw_moban_with_fs2(["moban"], folder, criterias) + self.run_moban_with_fs(["moban"], folder, criterias) def _raw_moban(self, args, folder, expected, output): base_dir = fs.path.join("tests", "regression_tests") @@ -82,39 +86,3 @@ def _raw_moban(self, args, folder, expected, output): status = filecmp.cmp(output, expected) os.unlink(output) assert status - - def _raw_moban_with_fs(self, args, folder, expected, output): - base_dir = fs.path.join("tests", "regression_tests") - os.chdir(fs.path.join(base_dir, folder)) - with patch.object(sys, "argv", args): - main() - _verify_content_with_fs(output, expected) - result = parse_fs_url(output) - os.unlink(result.resource) # delete the zip file - - def _raw_moban_with_fs2(self, args, folder, criterias): - base_dir = fs.path.join("tests", "regression_tests") - os.chdir(fs.path.join(base_dir, folder)) - with patch.object(sys, "argv", args): - main() - - for output, expected in criterias: - _verify_content_with_fs(output, expected) - result = parse_fs_url(output) - os.unlink(result.resource) # delete the zip file - - def tearDown(self): - if os.path.exists(".moban.hashes"): - os.unlink(".moban.hashes") - os.chdir(self.current) - - -def _verify_content(file_name, expected): - with open(file_name, "r") as f: - content = f.read() - eq_(content, expected) - - -def _verify_content_with_fs(file_name, expected): - content = file_system.read_unicode(file_name) - eq_(content, expected) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..986951ed --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,64 @@ +import os +import sys +from textwrap import dedent + +from mock import patch +from moban import file_system +from moban.main import main +from nose.tools import eq_ +from fs.opener.parse import parse_fs_url + + +def verify_content(file_name, expected): + with open(file_name, "r") as f: + content = f.read() + eq_(content, expected) + + +def verify_content_with_fs(file_name, expected): + content = file_system.read_unicode(file_name) + eq_(content, expected) + + +def run_moban(args, folder, criterias): + with patch.object(sys, "argv", args): + main() + for output, expected in criterias: + verify_content(output, expected) + os.unlink(output) + + +def run_moban_with_fs(args, folder, criterias): + with patch.object(sys, "argv", args): + main() + + for output, expected in criterias: + verify_content_with_fs(output, expected) + result = parse_fs_url(output) + os.unlink(result.resource) # delete the zip file + + +class Docs(object): + def setUp(self): + self.current = os.getcwd() + self.base_folder = "docs" + + def tearDown(self): + if os.path.exists(".moban.hashes"): + os.unlink(".moban.hashes") + os.chdir(self.current) + + def run_moban(self, moban_cli, working_directory, assertions): + os.chdir(os.path.join(self.base_folder, working_directory)) + run_moban(moban_cli, None, assertions) + + def run_moban_with_fs(self, moban_cli, working_directory, assertions): + os.chdir(os.path.join(self.base_folder, working_directory)) + run_moban_with_fs(moban_cli, None, assertions) + + +def custom_dedent(long_texts): + refined = dedent(long_texts) + if refined.startswith("\n"): + refined = refined[1:] + return refined From 5d3e2ae11b56019adddfe0bdbb7bec0a977958cb Mon Sep 17 00:00:00 2001 From: jaska Date: Wed, 21 Aug 2019 19:25:57 +0100 Subject: [PATCH 161/167] Code cleanning (#318) * :shirt: minor update * :hammer: use multi_fs instance to iterate file instead of an independent file system instance * :wheelchair: more logging and improve code coherence * :tractor: update code coherence --- moban/main.py | 21 +++++++++++-- moban/mobanfile/__init__.py | 44 +++++++------------------- moban/mobanfile/targets.py | 28 +++++++++++++++++ moban/mobanfile/templates.py | 52 +++++++++++++------------------ moban/utils.py | 2 -- tests/mobanfile/test_mobanfile.py | 32 ------------------- tests/test_main.py | 42 ++++++++++++++++++++++--- 7 files changed, 118 insertions(+), 103 deletions(-) diff --git a/moban/main.py b/moban/main.py index ae01553c..01ad8171 100644 --- a/moban/main.py +++ b/moban/main.py @@ -13,7 +13,15 @@ import argparse import logging.config -from moban import core, plugins, reporter, constants, mobanfile, exceptions +from moban import ( + core, + plugins, + reporter, + constants, + mobanfile, + exceptions, + file_system, +) from moban._version import __version__ from moban.hashstore import HASH_STORE from moban.data_loaders.manager import merge, load_data @@ -37,7 +45,7 @@ def main(): moban_file = options[constants.LABEL_MOBANFILE] load_engine_factory_and_engines() # Error: jinja2 if removed if moban_file is None: - moban_file = mobanfile.find_default_moban_file() + moban_file = find_default_moban_file() if moban_file: try: count = handle_moban_file(moban_file, options) @@ -226,5 +234,14 @@ def handle_command_line(options): return exit_code +def find_default_moban_file(): + for moban_file in constants.DEFAULT_MOBAN_FILES: + if file_system.exists(moban_file): + break + else: + moban_file = None + return moban_file + + def load_engine_factory_and_engines(): plugins.make_sure_all_pkg_are_loaded() diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index 4c0d1cd4..f3ae753d 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -1,27 +1,26 @@ import os import re import sys +import logging from collections import OrderedDict -from moban import core, reporter, constants, file_system +from moban import core, reporter, constants from lml.utils import do_import from moban.utils import verify_the_existence_of_directories from moban.deprecated import handle_copy, handle_requires -from moban.mobanfile.targets import parse_targets, extract_group_targets +from moban.mobanfile.targets import ( + parse_targets, + extract_target, + extract_group_targets, +) from moban.core.moban_factory import expand_template_directories from moban.data_loaders.manager import merge - -def find_default_moban_file(): - for moban_file in constants.DEFAULT_MOBAN_FILES: - if file_system.exists(moban_file): - break - else: - moban_file = None - return moban_file +LOG = logging.getLogger(__name__) def handle_moban_file_v1(moban_file_configurations, command_line_options): + LOG.info("handling moban file") merged_options = None targets = moban_file_configurations.get(constants.LABEL_TARGETS, []) @@ -47,6 +46,7 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): if plugins_dirs: handle_plugin_dirs(plugins_dirs) + # deprecated requires = moban_file_configurations.get(constants.LABEL_REQUIRES) if requires: handle_requires(requires) @@ -86,6 +86,7 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): def handle_targets(merged_options, targets): + LOG.info("handling targets") list_of_templating_parameters = parse_targets(merged_options, targets) jobs_for_each_engine = OrderedDict() @@ -123,6 +124,7 @@ def handle_targets(merged_options, targets): def handle_plugin_dirs(plugin_dirs): + LOG.info("handling plugin dirs {}".format(",".join(plugin_dirs))) for plugin_dir in plugin_dirs: plugin_path = os.path.normcase( os.path.dirname(os.path.abspath(plugin_dir)) @@ -135,25 +137,3 @@ def handle_plugin_dirs(plugin_dirs): for plugin in plugins: plugin_module = os.path.basename(plugin_dir) + "." + plugin do_import(plugin_module) - - -def extract_target(options): - template = options.get(constants.LABEL_TEMPLATE) - config = options.get(constants.LABEL_CONFIG) - output = options.get(constants.LABEL_OUTPUT) - result = [] - if template: - if output is None: - raise Exception( - "Please specify a output file name for %s." % template - ) - if config: - result = { - constants.LABEL_TEMPLATE: template, - constants.LABEL_CONFIG: config, - constants.LABEL_OUTPUT: output, - } - - else: - result = {output: template} - return result diff --git a/moban/mobanfile/targets.py b/moban/mobanfile/targets.py index 2ec64a5d..161de814 100644 --- a/moban/mobanfile/targets.py +++ b/moban/mobanfile/targets.py @@ -1,9 +1,36 @@ import uuid +import logging from moban import core, reporter, constants, exceptions from moban.definitions import TemplateTarget from moban.mobanfile.templates import handle_template +LOG = logging.getLogger(__name__) + + +def extract_target(options): + template, config, output = ( + options.get(constants.LABEL_TEMPLATE), + options.get(constants.LABEL_CONFIG), + options.get(constants.LABEL_OUTPUT), + ) + result = [] + if template: + if output is None: + raise Exception( + "Please specify a output file name for %s." % template + ) + if config: + result = { + constants.LABEL_TEMPLATE: template, + constants.LABEL_CONFIG: config, + constants.LABEL_OUTPUT: output, + } + + else: + result = {output: template} + return result + def extract_group_targets(group, targets): for target in targets: @@ -18,6 +45,7 @@ def extract_group_targets(group, targets): def parse_targets(options, targets): + LOG.info("paring targets..") for target in targets: if constants.LABEL_OUTPUT in target: for template_target in _handle_explicit_target(options, target): diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 73240e38..4042ac0e 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -2,20 +2,20 @@ from moban import reporter, file_system -log = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) def handle_template(template_file, output, template_dirs): - log.info("handling %s" % template_file) + LOG.info("handling %s" % template_file) template_file = file_system.to_unicode(template_file) multi_fs = file_system.get_multi_fs(template_dirs) if template_file.endswith("**"): source_dir = template_file[:-3] - if multi_fs.exists(source_dir): - src_path = multi_fs.geturl(source_dir, purpose="fs") + _, fs = multi_fs.which(source_dir) + if fs: for a_triple in _listing_directory_files_recusively( - source_dir, src_path, output + fs, source_dir, output ): yield a_triple else: @@ -23,50 +23,42 @@ def handle_template(template_file, output, template_dirs): "{0} cannot be found".format(template_file) ) else: - if not multi_fs.exists(template_file): + _, fs = multi_fs.which(template_file) + if fs is None: reporter.report_error_message( "{0} cannot be found".format(template_file) ) - elif multi_fs.isdir(template_file): - template_file_on_disk = multi_fs.geturl( - template_file, purpose="fs" - ) - for a_triple in _list_dir_files( - template_file, template_file_on_disk, output - ): + elif fs.isdir(template_file): + for a_triple in _list_dir_files(fs, template_file, output): yield a_triple else: template_type = _get_template_type(template_file) yield (template_file, output, template_type) -def _list_dir_files(source, actual_source_path, dest): - for file_name in file_system.list_dir(actual_source_path): - if file_system.is_file( - file_system.url_join(actual_source_path, file_name) - ): - # please note jinja2 does NOT like windows path - # hence the following statement looks like cross platform - # src_file_under_dir = os.path.join(source, file_name) - # but actually it breaks windows instead. - src_file_under_dir = "%s/%s" % (source, file_name) - +def _list_dir_files(fs, source, dest): + for file_name in fs.listdir(source): + # please note jinja2 does NOT like windows path + # hence the following statement looks like cross platform + # src_file_under_dir = os.path.join(source, file_name) + # but actually it breaks windows instead. + src_file_under_dir = "%s/%s" % (source, file_name) + if fs.isfile(src_file_under_dir): dest_file_under_dir = dest + "/" + file_name template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) -def _listing_directory_files_recusively(source, actual_source_path, dest): - for file_name in file_system.list_dir(actual_source_path): +def _listing_directory_files_recusively(fs, source, dest): + for file_name in fs.listdir(source): src_file_under_dir = source + "/" + file_name dest_file_under_dir = dest + "/" + file_name - real_src_file = file_system.url_join(actual_source_path, file_name) - if file_system.is_file(real_src_file): + if fs.isfile(src_file_under_dir): template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) - elif file_system.is_dir(real_src_file): + elif fs.isdir(src_file_under_dir): for a_triple in _listing_directory_files_recusively( - src_file_under_dir, real_src_file, dest_file_under_dir + fs, src_file_under_dir, dest_file_under_dir ): yield a_triple diff --git a/moban/utils.py b/moban/utils.py index d4e4aa11..98672fed 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,12 +1,10 @@ import os -import sys import errno import logging from moban import constants, exceptions, file_system LOG = logging.getLogger(__name__) -PY2 = sys.version_info[0] == 2 def mkdir_p(path): diff --git a/tests/mobanfile/test_mobanfile.py b/tests/mobanfile/test_mobanfile.py index 49552c2e..db8c11b7 100644 --- a/tests/mobanfile/test_mobanfile.py +++ b/tests/mobanfile/test_mobanfile.py @@ -4,38 +4,6 @@ from moban.definitions import TemplateTarget -class TestFinder: - def setUp(self): - self.patcher = patch("moban.file_system.exists") - self.fake_file_existence = self.patcher.start() - self.fake_file_existence.__name__ = "fake" - self.fake_file_existence.__module__ = "fake" - - def tearDown(self): - self.patcher.stop() - - def test_moban_yml(self): - self.fake_file_existence.return_value = True - from moban.mobanfile import find_default_moban_file - - actual = find_default_moban_file() - eq_(".moban.yml", actual) - - def test_moban_yaml(self): - self.fake_file_existence.side_effect = [False, True] - from moban.mobanfile import find_default_moban_file - - actual = find_default_moban_file() - eq_(".moban.yaml", actual) - - def test_no_moban_file(self): - self.fake_file_existence.side_effect = [False, False] - from moban.mobanfile import find_default_moban_file - - actual = find_default_moban_file() - assert actual is None - - @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_handle_targets(fake_renderer): from moban.mobanfile import handle_targets diff --git a/tests/test_main.py b/tests/test_main.py index 3f8b27db..96adba08 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -4,7 +4,7 @@ import moban.exceptions as exceptions from mock import patch -from nose.tools import raises, assert_raises +from nose.tools import eq_, raises, assert_raises class TestException: @@ -123,7 +123,7 @@ def test_double_underscore_main( class TestExitCodes: @raises(SystemExit) @patch("moban.main.handle_moban_file") - @patch("moban.mobanfile.find_default_moban_file") + @patch("moban.main.find_default_moban_file") def test_has_many_files_with_exit_code( self, fake_find_file, fake_moban_file ): @@ -136,7 +136,7 @@ def test_has_many_files_with_exit_code( @raises(SystemExit) @patch("moban.main.handle_command_line") - @patch("moban.mobanfile.find_default_moban_file") + @patch("moban.main.find_default_moban_file") def test_handle_single_change_with_exit_code( self, fake_find_file, fake_command_line ): @@ -148,7 +148,7 @@ def test_handle_single_change_with_exit_code( main() @patch("moban.main.handle_moban_file") - @patch("moban.mobanfile.find_default_moban_file") + @patch("moban.main.find_default_moban_file") def test_has_many_files(self, fake_find_file, fake_moban_file): fake_find_file.return_value = "abc" fake_moban_file.return_value = 1 @@ -158,7 +158,7 @@ def test_has_many_files(self, fake_find_file, fake_moban_file): main() @patch("moban.main.handle_command_line") - @patch("moban.mobanfile.find_default_moban_file") + @patch("moban.main.find_default_moban_file") def test_handle_single_change(self, fake_find_file, fake_command_line): fake_find_file.return_value = None fake_command_line.return_value = 1 @@ -166,3 +166,35 @@ def test_handle_single_change(self, fake_find_file, fake_command_line): with patch.object(sys, "argv", ["moban"]): main() + + +class TestFinder: + def setUp(self): + self.patcher = patch("moban.file_system.exists") + self.fake_file_existence = self.patcher.start() + self.fake_file_existence.__name__ = "fake" + self.fake_file_existence.__module__ = "fake" + + def tearDown(self): + self.patcher.stop() + + def test_moban_yml(self): + self.fake_file_existence.return_value = True + from moban.main import find_default_moban_file + + actual = find_default_moban_file() + eq_(".moban.yml", actual) + + def test_moban_yaml(self): + self.fake_file_existence.side_effect = [False, True] + from moban.main import find_default_moban_file + + actual = find_default_moban_file() + eq_(".moban.yaml", actual) + + def test_no_moban_file(self): + self.fake_file_existence.side_effect = [False, False] + from moban.main import find_default_moban_file + + actual = find_default_moban_file() + assert actual is None From df830b62396efc24fd0ef29d2e681e3882589fcf Mon Sep 17 00:00:00 2001 From: jaska Date: Thu, 22 Aug 2019 19:21:48 +0100 Subject: [PATCH 162/167] :bug: update templated count (#320) * :bug: update templated count * :bug: fix a typeo --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- moban/core/moban_factory.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 89283248..f62cf653 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,6 +3,6 @@ Before raising the PR, here is a check list: - [ ] have you written unit tests for your code changes? - [ ] have you updated the change log? - [ ] can someone else understand your changes without your explanation? -- [ ] are you pround of your code changes? +- [ ] are you proud of your code changes? - [ ] please add your name and github link to contributors.rst in alphabetical order. diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index bb63c919..adc4c1b1 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -84,7 +84,6 @@ def number_of_templated_files(self): return self.templated_count def render_to_file(self, template_file, data_file, output_file): - self.file_count = 1 template_file = file_system.to_unicode(template_file) data = self.context.get_data(data_file) template = self.engine.get_template(template_file) @@ -106,7 +105,6 @@ def render_to_file(self, template_file, data_file, output_file): def render_string_to_file( self, template_in_string, data_file, output_file ): - self.file_count = 1 template = self.engine.get_template_from_string(template_in_string) template_abs_path = template_in_string[:10] + "..." data = self.context.get_data(data_file) From 930a34fd2367d251354a69c85ebc2662b9503f25 Mon Sep 17 00:00:00 2001 From: jaska Date: Fri, 23 Aug 2019 17:34:37 +0100 Subject: [PATCH 163/167] Extended override (#321) * :sparkles: Allow mobanfile to include data from arbitrary config files. resolves #126 * :microscope: test cases for #126 * :microscope: more test cases * :shirt: update yaml styles * :books: update docs --- .moban.cd/changelog.yml | 5 +- CHANGELOG.rst | 8 +- docs/level-3-data-override/README.rst | 27 ++++++- moban/data_loaders/manager.py | 19 ++++- tests/data_loaders/test_overrides.py | 80 +++++++++++++++++++ tests/fixtures/child.yaml | 2 +- tests/fixtures/config/base.yaml | 2 +- tests/fixtures/issue_126/config/A.yaml | 1 + tests/fixtures/issue_126/config/B.yaml | 1 + .../issue_126/config/multi-key-A.yaml | 2 + .../issue_126/config/multi-key-B.yaml | 2 + .../issue_126/config/multi-key-config.yaml | 4 + tests/fixtures/issue_126/config/nested-A.yaml | 6 ++ tests/fixtures/issue_126/config/nested-B.yaml | 7 ++ tests/fixtures/issue_126/config_BA.yaml | 4 + .../issue_126/multi-key-config-override.yaml | 5 ++ .../fixtures/issue_126/multi-key-config.yaml | 4 + tests/fixtures/issue_126/raspberry.yaml | 5 ++ tests/fixtures/issue_126/the_config.yaml | 4 + 19 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 tests/data_loaders/test_overrides.py create mode 100644 tests/fixtures/issue_126/config/A.yaml create mode 100644 tests/fixtures/issue_126/config/B.yaml create mode 100644 tests/fixtures/issue_126/config/multi-key-A.yaml create mode 100644 tests/fixtures/issue_126/config/multi-key-B.yaml create mode 100644 tests/fixtures/issue_126/config/multi-key-config.yaml create mode 100644 tests/fixtures/issue_126/config/nested-A.yaml create mode 100644 tests/fixtures/issue_126/config/nested-B.yaml create mode 100644 tests/fixtures/issue_126/config_BA.yaml create mode 100644 tests/fixtures/issue_126/multi-key-config-override.yaml create mode 100644 tests/fixtures/issue_126/multi-key-config.yaml create mode 100644 tests/fixtures/issue_126/raspberry.yaml create mode 100644 tests/fixtures/issue_126/the_config.yaml diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 87fcf204..4bf8adf1 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -2,10 +2,13 @@ name: moban organisation: moremoban releases: - changes: - - action: Updated + - action: Added details: - "`#205`: support `pyFilesystem2 `_" - "`#185`: -d will enable moban application logging for development" + - "`#126`: Allow mobanfile to include data from arbitrary config files" + - action: Updated + details: - "`#275`: fix moban 0.4.5 test failures on openSUSE Tumbleweed" date: tba version: 0.6.0 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aae78ac8..c1943cc8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,13 +4,19 @@ Change log 0.6.0 - tba -------------------------------------------------------------------------------- -Updated +Added ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #. `#205 `_: support `pyFilesystem2 `_ #. `#185 `_: -d will enable moban application logging for development +#. `#126 `_: Allow mobanfile to + include data from arbitrary config files + +Updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + #. `#275 `_: fix moban 0.4.5 test failures on openSUSE Tumbleweed diff --git a/docs/level-3-data-override/README.rst b/docs/level-3-data-override/README.rst index c163c38a..66109d58 100644 --- a/docs/level-3-data-override/README.rst +++ b/docs/level-3-data-override/README.rst @@ -1,7 +1,8 @@ Level 3: data override ================================================================================ -What `moban` brings on the table is data inheritance by introducing `overrides` key word in the yaml file:: +What `moban` brings on the table is data inheritance by introducing `overrides` +key word in the yaml file:: overrides: data.base.yaml .... @@ -30,3 +31,27 @@ command to launch it: shijie ========footer============ + + +New development +================================================================================ + +Since verison 0.6.0, `overrides` syntax support two more use cases: + +1 override more than one configuration file +--------------------------------------------- + +For example:: + + overrides: + - config-file-a.yaml + - config-file-b.yaml + +2 override more than one configuration file +--------------------------------------------- + +For example:: + + overrides: + - config-file-a.yaml: keya + - config-file-b.yaml: keyb diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index 77724707..6c7a9097 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + import moban.data_loaders.yaml # noqa: F401 import moban.data_loaders.json_loader # noqa: F401 from moban import constants, file_system @@ -28,11 +30,20 @@ def load_data(base_dir, file_name): abs_file_path = search_file(base_dir, file_name) data = LOADER.get_data(abs_file_path) if data is not None: - parent_data = None + parent_data = OrderedDict() if base_dir and constants.LABEL_OVERRIDES in data: - parent_data = load_data( - base_dir, data.pop(constants.LABEL_OVERRIDES) - ) + overrides = data.pop(constants.LABEL_OVERRIDES) + if not isinstance(overrides, list): + overrides = [overrides] + for parent_file in overrides: + file_name, key = parent_file, None + if ":" in parent_file: + file_name, key = parent_file.split(":") + child_data = load_data(base_dir, file_name) + if data: + if key: + child_data = OrderedDict({key: child_data[key]}) + parent_data = merge(parent_data, child_data) if parent_data: return merge(data, parent_data) else: diff --git a/tests/data_loaders/test_overrides.py b/tests/data_loaders/test_overrides.py new file mode 100644 index 00000000..d9b190d3 --- /dev/null +++ b/tests/data_loaders/test_overrides.py @@ -0,0 +1,80 @@ +import os + +from nose.tools import eq_ +from moban.data_loaders.manager import load_data + + +def test_overrides_a_list_of_config_files(): + base_dir = os.path.join("tests", "fixtures", "issue_126") + config_dir = os.path.join(base_dir, "config") + actual = load_data(config_dir, os.path.join(base_dir, "the_config.yaml")) + expected = [ + ("key", "value"), + ("key_from_a", "apple"), + ("key_from_b", "bee"), + ] + for item, expected_item in zip(actual.items(), expected): + eq_(item, expected_item) + + +def test_overrides_ignores_override_sequence(): + base_dir = os.path.join("tests", "fixtures", "issue_126") + config_dir = os.path.join(base_dir, "config") + actual = load_data(config_dir, os.path.join(base_dir, "the_config.yaml")) + expected = [ + ("key", "value"), + ("key_from_a", "apple"), + ("key_from_b", "bee"), + ] + for item, expected_item in zip(actual.items(), expected): + eq_(item, expected_item) + + +def test_overrides_select_keys_from_parent_files(): + base_dir = os.path.join("tests", "fixtures", "issue_126") + config_dir = os.path.join(base_dir, "config") + actual = load_data( + config_dir, os.path.join(base_dir, "multi-key-config.yaml") + ) + expected = [ + ("cat", "from config"), + ("alpha", "from a"), + ("beta", "from b"), + ] + for item, expected_item in zip(actual.items(), expected): + eq_(item, expected_item) + + +def test_overrides_select_keys(): + base_dir = os.path.join("tests", "fixtures", "issue_126") + config_dir = os.path.join(base_dir, "config") + actual = load_data( + config_dir, os.path.join(base_dir, "multi-key-config-override.yaml") + ) + expected = [ + ("alpha", "from config"), + ("cat", "from config"), + ("beta", "from b"), + ] + for item, expected_item in zip(actual.items(), expected): + eq_(item, expected_item) + + +def test_overrides_nested_keys(): + base_dir = os.path.join("tests", "fixtures", "issue_126") + config_dir = os.path.join(base_dir, "config") + actual = load_data(config_dir, os.path.join(base_dir, "raspberry.yaml")) + expected = { + "raspberry": { + "other": "OpenGL 3.0", + "version": 4, + "memory": "4GB", + "core": "quad", + "wifi": "2.5 & 5.0 GHz", + "USB": 3.0, + "Bluetooth": 5.0, + }, + "tessel": {"version": 2, "USB": "micro", "wifi": "802.11gn"}, + } + + eq_(dict(actual), expected) diff --git a/tests/fixtures/child.yaml b/tests/fixtures/child.yaml index 4b082225..ed31c83b 100644 --- a/tests/fixtures/child.yaml +++ b/tests/fixtures/child.yaml @@ -1,2 +1,2 @@ +key: 'hello world' overrides: base.yaml -key: hello world diff --git a/tests/fixtures/config/base.yaml b/tests/fixtures/config/base.yaml index 0270c58b..1ef517a0 100644 --- a/tests/fixtures/config/base.yaml +++ b/tests/fixtures/config/base.yaml @@ -1,2 +1,2 @@ -key: to be overriden +key: 'to be overriden' pass: ox diff --git a/tests/fixtures/issue_126/config/A.yaml b/tests/fixtures/issue_126/config/A.yaml new file mode 100644 index 00000000..3ec33513 --- /dev/null +++ b/tests/fixtures/issue_126/config/A.yaml @@ -0,0 +1 @@ +key_from_a: apple diff --git a/tests/fixtures/issue_126/config/B.yaml b/tests/fixtures/issue_126/config/B.yaml new file mode 100644 index 00000000..9b200839 --- /dev/null +++ b/tests/fixtures/issue_126/config/B.yaml @@ -0,0 +1 @@ +key_from_b: bee diff --git a/tests/fixtures/issue_126/config/multi-key-A.yaml b/tests/fixtures/issue_126/config/multi-key-A.yaml new file mode 100644 index 00000000..79765e75 --- /dev/null +++ b/tests/fixtures/issue_126/config/multi-key-A.yaml @@ -0,0 +1,2 @@ +alpha: 'from a' +beta: 'from a' diff --git a/tests/fixtures/issue_126/config/multi-key-B.yaml b/tests/fixtures/issue_126/config/multi-key-B.yaml new file mode 100644 index 00000000..4ede311c --- /dev/null +++ b/tests/fixtures/issue_126/config/multi-key-B.yaml @@ -0,0 +1,2 @@ +alpha: 'from b' +beta: 'from b' diff --git a/tests/fixtures/issue_126/config/multi-key-config.yaml b/tests/fixtures/issue_126/config/multi-key-config.yaml new file mode 100644 index 00000000..43973344 --- /dev/null +++ b/tests/fixtures/issue_126/config/multi-key-config.yaml @@ -0,0 +1,4 @@ +cat: 'from config' +overrides: + - multi-key-A.yaml:alpha + - multi-key-B.yaml:beta diff --git a/tests/fixtures/issue_126/config/nested-A.yaml b/tests/fixtures/issue_126/config/nested-A.yaml new file mode 100644 index 00000000..9529fb1e --- /dev/null +++ b/tests/fixtures/issue_126/config/nested-A.yaml @@ -0,0 +1,6 @@ +raspberry: + core: quad + memory: 4GB + version: 4 +tessel: + version: 2 diff --git a/tests/fixtures/issue_126/config/nested-B.yaml b/tests/fixtures/issue_126/config/nested-B.yaml new file mode 100644 index 00000000..5c48e1a3 --- /dev/null +++ b/tests/fixtures/issue_126/config/nested-B.yaml @@ -0,0 +1,7 @@ +raspberry: + Bluetooth: 5.0 + USB: 3.0 + wifi: '2.5 & 5.0 GHz' +tessel: + USB: micro + wifi: 802.11gn diff --git a/tests/fixtures/issue_126/config_BA.yaml b/tests/fixtures/issue_126/config_BA.yaml new file mode 100644 index 00000000..823afb2b --- /dev/null +++ b/tests/fixtures/issue_126/config_BA.yaml @@ -0,0 +1,4 @@ +key: value +overrides: + - B.yaml + - A.yaml diff --git a/tests/fixtures/issue_126/multi-key-config-override.yaml b/tests/fixtures/issue_126/multi-key-config-override.yaml new file mode 100644 index 00000000..0f9c8024 --- /dev/null +++ b/tests/fixtures/issue_126/multi-key-config-override.yaml @@ -0,0 +1,5 @@ +alpha: 'from config' +cat: 'from config' +overrides: + - multi-key-A.yaml:alpha + - multi-key-B.yaml:beta diff --git a/tests/fixtures/issue_126/multi-key-config.yaml b/tests/fixtures/issue_126/multi-key-config.yaml new file mode 100644 index 00000000..43973344 --- /dev/null +++ b/tests/fixtures/issue_126/multi-key-config.yaml @@ -0,0 +1,4 @@ +cat: 'from config' +overrides: + - multi-key-A.yaml:alpha + - multi-key-B.yaml:beta diff --git a/tests/fixtures/issue_126/raspberry.yaml b/tests/fixtures/issue_126/raspberry.yaml new file mode 100644 index 00000000..feadf485 --- /dev/null +++ b/tests/fixtures/issue_126/raspberry.yaml @@ -0,0 +1,5 @@ +overrides: + - nested-A.yaml + - nested-B.yaml +raspberry: + other: 'OpenGL 3.0' diff --git a/tests/fixtures/issue_126/the_config.yaml b/tests/fixtures/issue_126/the_config.yaml new file mode 100644 index 00000000..d56563ff --- /dev/null +++ b/tests/fixtures/issue_126/the_config.yaml @@ -0,0 +1,4 @@ +key: value +overrides: + - A.yaml + - B.yaml From 9d28ccd21358d42aa1d3a8596dfee9a988bd4167 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 23 Aug 2019 18:26:41 +0100 Subject: [PATCH 164/167] :books: mentioned about git commit message emojis. fix #279 --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 434e0276..68e90e1a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,3 +48,7 @@ When you enable travis-ci on your own account, you shall see travis-ci running a 6. Commit with a suitable message `git commit -m " Changes made "` 7. Push `git push origin branch_name` 8. Go to the Github Repository and create a pull request to the dev branch + +## Commit messages + +We use git emojis for commit messages. Please read [the guide](https://github.com/slashsBin/styleguide-git-commit-message). From ff18b416be4e42f7aa4a13c37731bfad56f07a06 Mon Sep 17 00:00:00 2001 From: jaska Date: Thu, 29 Aug 2019 07:04:59 +0100 Subject: [PATCH 165/167] -D hello=world is added. (#324) * :sparkles: -D is added. resolves #256 * :books: update readme * :newspaper: add missing file. related to #256 * :wheelchair: better help texts --- .moban.cd/changelog.yml | 1 + CHANGELOG.rst | 2 + README.rst | 69 ++++++++++++++++-------------- docs/level-1-jinja2-cli/README.rst | 8 ++++ moban/constants.py | 2 + moban/core/context.py | 13 ++++-- moban/main.py | 23 +++++++++- moban/program_options.py | 3 ++ tests/test_docs.py | 6 +++ 9 files changed, 90 insertions(+), 37 deletions(-) create mode 100644 moban/program_options.py diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 4bf8adf1..28eec0d5 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -7,6 +7,7 @@ releases: - "`#205`: support `pyFilesystem2 `_" - "`#185`: -d will enable moban application logging for development" - "`#126`: Allow mobanfile to include data from arbitrary config files" + - "`#256`: jinja2-cli parity: '-D hello=world' to define custom variable on cli" - action: Updated details: - "`#275`: fix moban 0.4.5 test failures on openSUSE Tumbleweed" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c1943cc8..4d6a6d92 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,8 @@ Added application logging for development #. `#126 `_: Allow mobanfile to include data from arbitrary config files +#. `#256 `_: jinja2-cli parity: + '-D hello=world' to define custom variable on cli Updated ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/README.rst b/README.rst index 48ed575f..04c5ebf9 100644 --- a/README.rst +++ b/README.rst @@ -175,38 +175,43 @@ Usage .. code-block:: bash - usage: moban [-h] [-cd CONFIGURATION_DIR] [-c CONFIGURATION] - [-td [TEMPLATE_DIR [TEMPLATE_DIR ...]]] [-t TEMPLATE] [-o OUTPUT] - [-f] [-m MOBANFILE] - [template] - - Yet another jinja2 cli command for static text generation - - positional arguments: - template string templates - - optional arguments: - -h, --help show this help message and exit - -cd CONFIGURATION_DIR, --configuration_dir CONFIGURATION_DIR - the directory for configuration file lookup - -c CONFIGURATION, --configuration CONFIGURATION - the dictionary file. if not present, moban - will try to use environment vars as data - -td [TEMPLATE_DIR [TEMPLATE_DIR ...]], --template_dir [TEMPLATE_DIR [TEMPLATE_DIR ...]] - the directories for template file lookup - -t TEMPLATE, --template TEMPLATE - the template file. this overrides any targets - defined in a custom moban file - -o OUTPUT, --output OUTPUT - the output file - --template_type TEMPLATE_TYPE - the template type, default is jinja2 - -f force moban to template all files despite of - .moban.hashes - --exit-code tell moban to change exit code - -m MOBANFILE, --mobanfile MOBANFILE - custom moban file - + usage: moban [-h] [-cd CONFIGURATION_DIR] [-c CONFIGURATION] + [-td [TEMPLATE_DIR [TEMPLATE_DIR ...]]] [-t TEMPLATE] [-o OUTPUT] + [--template_type TEMPLATE_TYPE] [-f] [--exit-code] [-m MOBANFILE] + [-g GROUP] [-v] [-d] [-D DEFINE [DEFINE ...]] + [template] + + Yet another jinja2 cli command for static text generation + + positional arguments: + template string templates + + optional arguments: + -h, --help show this help message and exit + -cd CONFIGURATION_DIR, --configuration_dir CONFIGURATION_DIR + the directory for configuration file lookup + -c CONFIGURATION, --configuration CONFIGURATION + the dictionary file + -td [TEMPLATE_DIR [TEMPLATE_DIR ...]], --template_dir [TEMPLATE_DIR [TEMPLATE_DIR ...]] + the directories for template file lookup + -t TEMPLATE, --template TEMPLATE + the template file + -o OUTPUT, --output OUTPUT + the output file + --template_type TEMPLATE_TYPE + the template type, default is jinja2 + -f force moban to template all files despite of + .moban.hashes + --exit-code tell moban to change exit code + -m MOBANFILE, --mobanfile MOBANFILE + custom moban file + -g GROUP, --group GROUP + a subset of targets + -v, --version show program's version number and exit + -d to show trace log + -D DEFINE [DEFINE ...], --define DEFINE [DEFINE ...] + to take in a list of VAR=VALUEs + Exit codes -------------------------------------------------------------------------------- diff --git a/docs/level-1-jinja2-cli/README.rst b/docs/level-1-jinja2-cli/README.rst index e8d4d673..845cad76 100644 --- a/docs/level-1-jinja2-cli/README.rst +++ b/docs/level-1-jinja2-cli/README.rst @@ -40,3 +40,11 @@ Then go to `docs/level-1-jinja2-cli`. here are different commands to evaluate it moban -t a.template because moban looks for `data.yml` by default + +As well, you can define your own variable: + +.. code-block:: bash + + moban -D hello=maailman -t a.template + +And when you check 'moban.output', you will find you have overwritten data.yaml. diff --git a/moban/constants.py b/moban/constants.py index 169cbe15..d4cc00f7 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -36,6 +36,8 @@ LABEL_FORCE = "force" LABEL_VERSION = "version" LABEL_GROUP = "group" +LABEL_DEFINE = "define" +CLI_DICT = "user_dict" DEFAULT_CONFIGURATION_DIRNAME = ".%s.cd" % PROGRAM_NAME DEFAULT_TEMPLATE_DIRNAME = ".%s.td" % PROGRAM_NAME diff --git a/moban/core/context.py b/moban/core/context.py index 367ec0d6..10ce96f0 100644 --- a/moban/core/context.py +++ b/moban/core/context.py @@ -1,6 +1,8 @@ import os +import copy -from moban import utils, reporter, exceptions +from moban import utils, reporter, constants, exceptions +from moban.program_options import OPTIONS from moban.data_loaders.manager import merge, load_data @@ -13,14 +15,17 @@ def __init__(self, context_dirs): ) def get_data(self, file_name): + custom_data = copy.deepcopy(OPTIONS[constants.CLI_DICT]) try: data = load_data(self.context_dirs, file_name) - merge(data, self.__cached_environ_variables) - return data + merge(custom_data, data) + merge(custom_data, self.__cached_environ_variables) + return custom_data except (IOError, exceptions.IncorrectDataInput) as exception: # If data file doesn't exist: # 1. Alert the user of their (potential) mistake # 2. Attempt to use environment vars as data reporter.report_warning_message(str(exception)) reporter.report_using_env_vars() - return self.__cached_environ_variables + merge(custom_data, self.__cached_environ_variables) + return custom_data diff --git a/moban/main.py b/moban/main.py index 01ad8171..e36f7b55 100644 --- a/moban/main.py +++ b/moban/main.py @@ -24,6 +24,7 @@ ) from moban._version import __version__ from moban.hashstore import HASH_STORE +from moban.program_options import OPTIONS from moban.data_loaders.manager import merge, load_data LOG = logging.getLogger() @@ -36,6 +37,10 @@ def main(): parser = create_parser() options = vars(parser.parse_args()) HASH_STORE.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE] + options[constants.CLI_DICT] = handle_custom_variables( + options.pop(constants.LABEL_DEFINE) + ) + OPTIONS.update(options) if options[constants.LABEL_DEBUG]: logging.basicConfig( format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", @@ -145,7 +150,13 @@ def create_parser(): action="store_true", dest=constants.LABEL_DEBUG, default=False, - help="to show debug trace", + help="to show trace log", + ) + parser.add_argument( + "-D", + "--%s" % constants.LABEL_DEFINE, + nargs="+", + help="to take a list of VAR=VALUEs", ) return parser @@ -245,3 +256,13 @@ def find_default_moban_file(): def load_engine_factory_and_engines(): plugins.make_sure_all_pkg_are_loaded() + + +def handle_custom_variables(list_of_definitions): + custom_data = {} + if list_of_definitions: + for definition in list_of_definitions: + key, value = definition.split("=") + custom_data[key] = value + + return custom_data diff --git a/moban/program_options.py b/moban/program_options.py new file mode 100644 index 00000000..efe0b0a7 --- /dev/null +++ b/moban/program_options.py @@ -0,0 +1,3 @@ +from moban import constants + +OPTIONS = {constants.CLI_DICT: {}} diff --git a/tests/test_docs.py b/tests/test_docs.py index 254ce95b..2a3217b9 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -11,6 +11,12 @@ def test_level_1(self): folder = "level-1-jinja2-cli" self._moban(folder, expected) + def test_level_1_custom_define(self): + expected = "maailman" + folder = "level-1-jinja2-cli" + args = ["moban", "-D", "hello=maailman", "-t", "a.template"] + self.run_moban(args, folder, [("moban.output", expected)]) + def test_level_2(self): expected = """ ========header============ From bf5ab523c119bde30f3220ddc4c6bb30105e10dd Mon Sep 17 00:00:00 2001 From: jaska Date: Mon, 9 Sep 2019 17:46:30 +0100 Subject: [PATCH 166/167] Debug option -v and -vv (#326) * :hammer: move version query to -V and give way for -vvv. #325 * :sparkles: support notrace, info and debug level verbose. fix #325 * :books: update readme * :hammer: give -d the original name space as define, not debug. #325 * :sparkles: enable minimal requirements testing as fs 2.4.11 has been released * :books: note about -V --- .moban.cd/changelog.yml | 3 +- .moban.cd/moban.yml | 2 +- .moban.d/moban_travis.yml.jj2 | 1 + .travis.yml | 1 + CHANGELOG.rst | 5 +-- README.rst | 12 +++---- min_requirements.txt | 2 +- moban/constants.py | 2 +- moban/main.py | 31 ++++++++++++------- requirements.txt | 2 +- rnd_requirements.txt | 1 - setup.py | 2 +- .../test_command_line_options.py | 4 +-- tests/test_docs.py | 2 +- 14 files changed, 40 insertions(+), 30 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 28eec0d5..ca4fd648 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -5,7 +5,8 @@ releases: - action: Added details: - "`#205`: support `pyFilesystem2 `_" - - "`#185`: -d will enable moban application logging for development" + - "`#185`: -v will enable moban application logging for development. And -V is for version." + - "`#325`: -vv show debug trace" - "`#126`: Allow mobanfile to include data from arbitrary config files" - "`#256`: jinja2-cli parity: '-D hello=world' to define custom variable on cli" - action: Updated diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 52eedfdb..bfcd1fdf 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -26,7 +26,7 @@ dependencies: - lml>=0.0.9 - appdirs>=1.4.3 - crayons>= 0.1.0 - - fs>=2.4.6 + - fs>=2.4.11 - jinja2-fsloader>=0.2.0 description: Yet another jinja2 cli command for static text generation scm_host: github.com diff --git a/.moban.d/moban_travis.yml.jj2 b/.moban.d/moban_travis.yml.jj2 index dc4c5c83..91ba2fd4 100644 --- a/.moban.d/moban_travis.yml.jj2 +++ b/.moban.d/moban_travis.yml.jj2 @@ -3,6 +3,7 @@ {%block extra_matrix %} env: - MINREQ=0 + - MINREQ=1 {%endblock%} {%block custom_python_versions%} diff --git a/.travis.yml b/.travis.yml index 65934ee3..47aad46d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ python: - 3.8-dev env: - MINREQ=0 + - MINREQ=1 stages: - lint diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4d6a6d92..d0c29c06 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,8 +9,9 @@ Added #. `#205 `_: support `pyFilesystem2 `_ -#. `#185 `_: -d will enable moban - application logging for development +#. `#185 `_: -v will enable moban + application logging for development. And -V is for version. +#. `#325 `_: -vv show debug trace #. `#126 `_: Allow mobanfile to include data from arbitrary config files #. `#256 `_: jinja2-cli parity: diff --git a/README.rst b/README.rst index 04c5ebf9..cd4fd545 100644 --- a/README.rst +++ b/README.rst @@ -178,7 +178,7 @@ Usage usage: moban [-h] [-cd CONFIGURATION_DIR] [-c CONFIGURATION] [-td [TEMPLATE_DIR [TEMPLATE_DIR ...]]] [-t TEMPLATE] [-o OUTPUT] [--template_type TEMPLATE_TYPE] [-f] [--exit-code] [-m MOBANFILE] - [-g GROUP] [-v] [-d] [-D DEFINE [DEFINE ...]] + [-g GROUP] [-V] [-v] [-D DEFINE [DEFINE ...]] [template] Yet another jinja2 cli command for static text generation @@ -207,11 +207,11 @@ Usage custom moban file -g GROUP, --group GROUP a subset of targets - -v, --version show program's version number and exit - -d to show trace log - -D DEFINE [DEFINE ...], --define DEFINE [DEFINE ...] - to take in a list of VAR=VALUEs - + -V, --version show program's version number and exit + -v show verbose + -d DEFINE [DEFINE ...], --define DEFINE [DEFINE ...] + to take a list of VAR=VALUEs + Exit codes -------------------------------------------------------------------------------- diff --git a/min_requirements.txt b/min_requirements.txt index 6c8b55b5..c3cac050 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -6,5 +6,5 @@ jinja2==2.7.1 lml==0.0.9 appdirs==1.4.3 crayons== 0.1.0 -fs==2.4.6 +fs==2.4.11 jinja2-fsloader==0.2.0 diff --git a/moban/constants.py b/moban/constants.py index d4cc00f7..8b86bb64 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -72,7 +72,7 @@ LABEL_DEST = "destination" LABEL_FORCE_TEMPLATE_TYPE = "force_template_type" LABEL_TEMPLATE_TYPES = "template_types" -LABEL_DEBUG = "debug" +LABEL_VERBOSE = "verbose" # error messages ERROR_DATA_FILE_NOT_FOUND = "Both %s and %s does not exist" diff --git a/moban/main.py b/moban/main.py index e36f7b55..d7872b1b 100644 --- a/moban/main.py +++ b/moban/main.py @@ -28,6 +28,7 @@ from moban.data_loaders.manager import merge, load_data LOG = logging.getLogger() +LOG_LEVEL = [logging.WARNING, logging.INFO, logging.DEBUG] def main(): @@ -41,11 +42,7 @@ def main(): options.pop(constants.LABEL_DEFINE) ) OPTIONS.update(options) - if options[constants.LABEL_DEBUG]: - logging.basicConfig( - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - level=logging.DEBUG, - ) + handle_verbose(options[constants.LABEL_VERBOSE]) moban_file = options[constants.LABEL_MOBANFILE] load_engine_factory_and_engines() # Error: jinja2 if removed @@ -140,20 +137,20 @@ def create_parser(): help="string templates", ) parser.add_argument( - "-v", + "-V", "--%s" % constants.LABEL_VERSION, action="version", version="%(prog)s {v}".format(v=__version__), ) parser.add_argument( - "-d", - action="store_true", - dest=constants.LABEL_DEBUG, - default=False, - help="to show trace log", + "-v", + action="count", + dest=constants.LABEL_VERBOSE, + default=0, + help="show verbose", ) parser.add_argument( - "-D", + "-d", "--%s" % constants.LABEL_DEFINE, nargs="+", help="to take a list of VAR=VALUEs", @@ -266,3 +263,13 @@ def handle_custom_variables(list_of_definitions): custom_data[key] = value return custom_data + + +def handle_verbose(verbose_level): + if verbose_level > len(LOG_LEVEL): + verbose_level = 3 + level = LOG_LEVEL[verbose_level] + logging.basicConfig( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + level=level, + ) diff --git a/requirements.txt b/requirements.txt index ff6ed355..8ad69281 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ jinja2>=2.7.1 lml>=0.0.9 appdirs>=1.4.3 crayons>= 0.1.0 -fs>=2.4.6 +fs>=2.4.11 jinja2-fsloader>=0.2.0 diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 79f046bb..9e76ec64 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,4 +1,3 @@ -https://github.com/moremoban/pyfilesystem2/archive/master.zip https://github.com/moremoban/moban-handlebars/archive/dev.zip diff --git a/setup.py b/setup.py index 94b3dc73..baed8541 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ "lml>=0.0.9", "appdirs>=1.4.3", "crayons>= 0.1.0", - "fs>=2.4.6", + "fs>=2.4.11", "jinja2-fsloader>=0.2.0", ] SETUP_COMMANDS = {} diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index bc732cca..302b99ff 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -436,7 +436,7 @@ def tearDown(self): @raises(SystemExit) def test_version_option(): - test_args = ["moban", "-v"] + test_args = ["moban", "-V"] with patch.object(sys, "argv", test_args): from moban.main import main @@ -446,7 +446,7 @@ def test_version_option(): @patch("logging.basicConfig") def test_debug_option(fake_config): fake_config.side_effect = [IOError("stop test")] - test_args = ["moban", "-d"] + test_args = ["moban", "-vv"] with patch.object(sys, "argv", test_args): from moban.main import main diff --git a/tests/test_docs.py b/tests/test_docs.py index 2a3217b9..2c711357 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -14,7 +14,7 @@ def test_level_1(self): def test_level_1_custom_define(self): expected = "maailman" folder = "level-1-jinja2-cli" - args = ["moban", "-D", "hello=maailman", "-t", "a.template"] + args = ["moban", "-d", "hello=maailman", "-t", "a.template"] self.run_moban(args, folder, [("moban.output", expected)]) def test_level_2(self): From c62ea15c4463001290777f10ddabfbbdf2e244a8 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 9 Sep 2019 17:58:29 +0100 Subject: [PATCH 167/167] :egg: :ferris_wheel: release 0.6.0 - PyFileSystem2 Support --- .moban.cd/changelog.yml | 2 +- .moban.cd/moban.yml | 2 +- CHANGELOG.rst | 2 +- docs/conf.py | 2 +- setup.py | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index ca4fd648..220de36d 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -12,7 +12,7 @@ releases: - action: Updated details: - "`#275`: fix moban 0.4.5 test failures on openSUSE Tumbleweed" - date: tba + date: 10.09.2019 version: 0.6.0 - changes: - action: Updated diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index bfcd1fdf..26bbdbc5 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -6,7 +6,7 @@ contact: wangc_2011@hotmail.com license: MIT version: 0.6.0 current_version: 0.6.0 -release: 0.5.0 +release: 0.6.0 branch: master master: index command_line_interface: "moban" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d0c29c06..11bc1140 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Change log ================================================================================ -0.6.0 - tba +0.6.0 - 10.09.2019 -------------------------------------------------------------------------------- Added diff --git a/docs/conf.py b/docs/conf.py index 24bc782d..211c3008 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,7 +27,7 @@ # The short X.Y version version = '0.6.0' # The full version, including alpha/beta/rc tags -release = '0.5.0' +release = '0.6.0' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index baed8541..0c6abfaf 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ "Yet another jinja2 cli command for static text generation" ) URL = "https://github.com/moremoban/moban" -DOWNLOAD_URL = "%s/archive/0.5.0.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.0.tar.gz" % URL FILES = ["README.rst", "CONTRIBUTORS.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -95,8 +95,8 @@ } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs moban v0.5.0 " + - "Find 0.5.0 in changelog for more details") +GS_COMMAND = ("gs moban v0.6.0 " + + "Find 0.6.0 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = (