diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..7f9d74f7 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +patreon: chfw diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d29a3ddb..f62cf653 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,8 @@ Before raising the PR, here is a check list: -1) have you written unit tests for your code changes? -2) have you run "make format"? -3) are you requesting to "dev"? -4) have you updated the change log? -5) do you think that you can understand your changes after 6 month? - 5.1) can someone else understand your changes without your explanation? -6) are you pround of your code changes? - 6.1) do you have the feeling of achievement? -7) please add your name and github link to contributors.rst in alphabetical order. \ No newline at end of file +- [ ] have you written unit tests for your code changes? +- [ ] have you updated the change log? +- [ ] can someone else understand your changes without your explanation? +- [ ] are you proud of your code changes? +- [ ] please add your name and github link to contributors.rst in alphabetical order. + diff --git a/.gitignore b/.gitignore index 5ef0acc0..95e2ceaf 100644 --- a/.gitignore +++ b/.gitignore @@ -506,6 +506,8 @@ cscope.in.out cscope.po.out +docs/deprecated-level-9-moban-dependency-as-pypi-package/mytravis.yml +docs/deprecated-level-10-moban-dependency-as-git-repo/mytravis.yml docs/level-1-jinja2-cli/testout docs/level-10-moban-dependency-as-git-repo/mytravis.yml docs/level-18-user-defined-template-types/b.output diff --git a/.isort.cfg b/.isort.cfg index d76a1394..25965d9b 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -2,7 +2,7 @@ line_length=79 # Ignore generated files skip=setup.py, moban/__init__.py -known_first_party=lml, crayons, jinja2, ruamel.yaml, mock, nose +known_first_party=fs, lml, crayons, jinja2, ruamel.yaml, mock, nose indent=' ' multi_line_output=3 length_sort=1 diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index bc1ba839..220de36d 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,19 @@ name: moban organisation: moremoban releases: + - changes: + - action: Added + details: + - "`#205`: support `pyFilesystem2 `_" + - "`#185`: -v will enable moban application logging for development. And -V is for version." + - "`#325`: -vv show debug trace" + - "`#126`: Allow mobanfile to include data from arbitrary config files" + - "`#256`: jinja2-cli parity: '-D hello=world' to define custom variable on cli" + - action: Updated + details: + - "`#275`: fix moban 0.4.5 test failures on openSUSE Tumbleweed" + date: 10.09.2019 + version: 0.6.0 - changes: - action: Updated details: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 2a3dca1f..26bbdbc5 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -4,9 +4,9 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.5.0 -current_version: 0.5.0 -release: 0.5.0 +version: 0.6.0 +current_version: 0.6.0 +release: 0.6.0 branch: master master: index command_line_interface: "moban" @@ -24,10 +24,10 @@ dependencies: - ruamel.yaml>=0.15.98;python_version == '3.8' - jinja2>=2.7.1 - lml>=0.0.9 - - appdirs>=1.2.0 + - appdirs>=1.4.3 - crayons>= 0.1.0 - - GitPython>=2.0.0 - - git-url-parse>=1.2.2 + - fs>=2.4.11 + - jinja2-fsloader>=0.2.0 description: Yet another jinja2 cli command for static text generation scm_host: github.com lint_command: make install_test format git-diff-check lint diff --git a/.moban.d/moban_gitignore.jj2 b/.moban.d/moban_gitignore.jj2 index 3ed4a68e..7f19ac2f 100644 --- a/.moban.d/moban_gitignore.jj2 +++ b/.moban.d/moban_gitignore.jj2 @@ -1,6 +1,8 @@ {% extends "gitignore.jj2" %} {% block extra %} +docs/deprecated-level-9-moban-dependency-as-pypi-package/mytravis.yml +docs/deprecated-level-10-moban-dependency-as-git-repo/mytravis.yml docs/level-1-jinja2-cli/testout docs/level-10-moban-dependency-as-git-repo/mytravis.yml docs/level-18-user-defined-template-types/b.output diff --git a/.travis.yml b/.travis.yml index 4f34a3e4..47aad46d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,9 +16,9 @@ env: - MINREQ=1 stages: - - test - lint - moban + - test .disable_global: &disable_global addons: false diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 21e34261..11bc1140 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,28 @@ Change log ================================================================================ +0.6.0 - 10.09.2019 +-------------------------------------------------------------------------------- + +Added +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#205 `_: support + `pyFilesystem2 `_ +#. `#185 `_: -v will enable moban + application logging for development. And -V is for version. +#. `#325 `_: -vv show debug trace +#. `#126 `_: Allow mobanfile to + include data from arbitrary config files +#. `#256 `_: jinja2-cli parity: + '-D hello=world' to define custom variable on cli + +Updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#275 `_: fix moban 0.4.5 test + failures on openSUSE Tumbleweed + 0.5.0 - 14.07.2019 -------------------------------------------------------------------------------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 434e0276..68e90e1a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,3 +48,7 @@ When you enable travis-ci on your own account, you shall see travis-ci running a 6. Commit with a suitable message `git commit -m " Changes made "` 7. Push `git push origin branch_name` 8. Go to the Github Repository and create a pull request to the dev branch + +## Commit messages + +We use git emojis for commit messages. Please read [the guide](https://github.com/slashsBin/styleguide-git-commit-message). diff --git a/Makefile b/Makefile index b7e5f2c3..5d9ead41 100644 --- a/Makefile +++ b/Makefile @@ -27,3 +27,10 @@ format: git diff black -l 79 tests git diff + +uml: + plantuml -tsvg -o ./images/ docs/*.uml + + +doc: uml + sphinx-build -b html docs build diff --git a/README.rst b/README.rst index 528b5aec..cd4fd545 100644 --- a/README.rst +++ b/README.rst @@ -25,13 +25,20 @@ moban - 模板 Yet another jinja2 cli command for static text generation **moban** brings the high performance template engine (JINJA2) for web into -static text generation. It is used in `pyexcel` and `coala` project to keep documentation -consistent across the documentations of individual libraries. +static text generation. It is used in `pyexcel` and `coala` project to keep +documentation consistent across the documentations of individual libraries. -Our vision is: any template, any data. Our current architecture enables moban -to plugin any python template engine: mako, handlebars, velocity, haml, slim and tornado -and to plugin any data format: json and yaml. Please look at our issues. We have many -more template engines and data format on the road map. +Our vision is: any template, any data in any location. Our current architecture +enables moban to plugin any python template engine: mako, handlebars, velocity, +haml, slim and tornado, to plugin any data format: json and yaml, and in +any location: zip, git, pypi package, s3, etc. Please +look at our issues. We have many more template engines and data format on the +road map. + +Documentation +================================================================================= + +All use cases are documented `here `_ Installation ================================================================================ @@ -93,114 +100,128 @@ moban.output will contain:: Please note that data.yml will take precedence over environment variables. -`the tutorial`_ has more use cases. - -.. _the tutorial: http://moban.readthedocs.org/en/latest/#tutorial - -Usage +Work with files in a git repo ================================================================================ -.. code-block:: bash +Please install `gitfs2 `_:: + $ pip install gitfs2 - usage: moban [-h] [-cd CONFIGURATION_DIR] [-c CONFIGURATION] - [-td [TEMPLATE_DIR [TEMPLATE_DIR ...]]] [-t TEMPLATE] [-o OUTPUT] - [-f] [-m MOBANFILE] - [template] - Yet another jinja2 cli command for static text generation - - positional arguments: - template string templates - - optional arguments: - -h, --help show this help message and exit - -cd CONFIGURATION_DIR, --configuration_dir CONFIGURATION_DIR - the directory for configuration file lookup - -c CONFIGURATION, --configuration CONFIGURATION - the dictionary file. if not present, moban - will try to use environment vars as data - -td [TEMPLATE_DIR [TEMPLATE_DIR ...]], --template_dir [TEMPLATE_DIR [TEMPLATE_DIR ...]] - the directories for template file lookup - -t TEMPLATE, --template TEMPLATE - the template file. this overrides any targets - defined in a custom moban file - -o OUTPUT, --output OUTPUT - the output file - --template_type TEMPLATE_TYPE - the template type, default is jinja2 - -f force moban to template all files despite of - .moban.hashes - --exit-code tell moban to change exit code - -m MOBANFILE, --mobanfile MOBANFILE - custom moban file - - -Exit codes --------------------------------------------------------------------------------- -By default: +And then you can do the following: -- 0 : no error -- 1 : error occured +.. code-block:: bash -With `--exit-code`: + $ moban -t 'git://github.com/moremoban/pypi-mobans.git!/templates/_version.py.jj2' \ + -c 'git://github.com/moremoban/pypi-mobans.git!/config/data.yml' \ + -o _version.py + Info: Found repo in /Users/jaska/Library/Caches/gitfs2/repos/pypi-mobans + Templating git://github.com/moremoban/pypi-mobans.git!/templates/_version.py.jj2 to _version.py + Templated 1 file. + $ cat _version.py + __version__ = "0.1.1rc3" + __author__ = "C.W." -- 0 : no changes -- 1 : has changes -- 2 : error occured -Built-in Filters +Work with files in a python package ================================================================================ -split_length --------------------------------------------------------------------------------- +Please install `pypifs `_:: -It breaks down the given string into a fixed length paragraph. Here is the syntax:: + $ pip install pypifs - {% for line in your_string | split_length(your_line_with) %} - {{line}} - {% endfor %} -It is used to keep changelog formatted in -`CHANGELOG.rst.jj2 in pypi-mobans project `_ +And then you can do the following: -github_expand --------------------------------------------------------------------------------- +.. code-block:: bash -It expands simple hashtags into github issues. Here is the syntax:: + $ moban -t 'pypi://pypi-mobans-pkg/resources/templates/_version.py.jj2' \ + -c 'pypi://pypi-mobans-pkg/resources/config/data.yml' \ + -o _version.py + Collecting pypi-mobans-pkg + .... + Installing collected packages: pypi-mobans-pkg + Successfully installed pypi-mobans-pkg-0.0.7 + Templating pypi://pypi-mobans-pkg/resources/templates/_version.py.jj2 to _version.py + Templated 1 file. + $ cat _version.py + __version__ = "0.1.1rc3" + __author__ = "C.W." - {{ your_github_string | github_expand }} +Work with S3 and other cloud based file systems +================================================================================ +Please install `fs-s3fs `_:: -It makes it easy to mention github reference in change log in all projects. Here is -the place it is applied: -`CHANGELOG.rst.jj2 in pypi-mobans project `_ + $ pip install fs-s3fs +.. code-block:: bash -Here is Grammar in the changelog.yml:: + $ moban -c s3://${client_id}:${client_secrect}@moremoban/s3data.yml \ + -o 'zip://my.zip!/moban.output' {{hello}} + $ unzip my.zip + $ cat moban.output + world - =============== ============================== - Syntax Meaning - =============== ============================== - `#1` moban issues 1 - `PR#1` moban pull request 1 - `pyexcel#1` other project issues 1 - `pyexcel#PR#1` other project pulll request 1 - =============== ============================== +Where the configuration sits in a s3 bucket, the output is a file in a zip. The content of s3data.yaml is:: -More details can be found in `moban's changelog.yml `_ + hello: world -`repr` --------------------------------------------------------------------------------- + +Usage +================================================================================ -Returns a single quoted string in the templated file +.. code-block:: bash -Built-in Tests -================================================================================ + usage: moban [-h] [-cd CONFIGURATION_DIR] [-c CONFIGURATION] + [-td [TEMPLATE_DIR [TEMPLATE_DIR ...]]] [-t TEMPLATE] [-o OUTPUT] + [--template_type TEMPLATE_TYPE] [-f] [--exit-code] [-m MOBANFILE] + [-g GROUP] [-V] [-v] [-D DEFINE [DEFINE ...]] + [template] + + Yet another jinja2 cli command for static text generation + + positional arguments: + template string templates + + optional arguments: + -h, --help show this help message and exit + -cd CONFIGURATION_DIR, --configuration_dir CONFIGURATION_DIR + the directory for configuration file lookup + -c CONFIGURATION, --configuration CONFIGURATION + the dictionary file + -td [TEMPLATE_DIR [TEMPLATE_DIR ...]], --template_dir [TEMPLATE_DIR [TEMPLATE_DIR ...]] + the directories for template file lookup + -t TEMPLATE, --template TEMPLATE + the template file + -o OUTPUT, --output OUTPUT + the output file + --template_type TEMPLATE_TYPE + the template type, default is jinja2 + -f force moban to template all files despite of + .moban.hashes + --exit-code tell moban to change exit code + -m MOBANFILE, --mobanfile MOBANFILE + custom moban file + -g GROUP, --group GROUP + a subset of targets + -V, --version show program's version number and exit + -v show verbose + -d DEFINE [DEFINE ...], --define DEFINE [DEFINE ...] + to take a list of VAR=VALUEs + -`exists` +Exit codes -------------------------------------------------------------------------------- +By default: + +- 0 : no error +- 1 : error occured -Test if a file exists or not +With `--exit-code`: + +- 0 : no changes +- 1 : has changes +- 2 : error occured diff --git a/docs/README.rst b/docs/README.rst index b4aff488..cfbe4a9f 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -22,7 +22,9 @@ This section covers the use cases for moban. Please check them out individually. #. `Force template type from moban file`_ #. `User defined template types`_ #. `Select a group target to run`_ - +#. `Template files in a zip or tar`_ +#. `Template copying from a zip to a zip`_ + .. _Jinja2 command line: level-1-jinja2-cli .. _Template inheritance: level-2-template-inheritance .. _Data override: level-3-data-override @@ -42,3 +44,5 @@ This section covers the use cases for moban. Please check them out individually. .. _Force template type from moban file: level-17-force-template-type-from-moban-file .. _User defined template types: level-18-user-defined-template-types .. _Select a group target to run: level-19-moban-a-sub-group-in-targets +.. _Template files in a zip or tar: level-20-templates-configs-in-zip-or-tar +.. _Template copying from a zip to a zip: level-21-copy-templates-into-an-alien-file-system diff --git a/docs/conf.py b/docs/conf.py index 4f768e04..211c3008 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,9 +25,9 @@ copyright = '2017-2019 Onni Software Ltd.' author = 'C. W.' # The short X.Y version -version = '0.5.0' +version = '0.6.0' # The full version, including alpha/beta/rc tags -release = '0.5.0' +release = '0.6.0' # -- General configuration --------------------------------------------------- @@ -69,6 +69,7 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} + # TODO: html_theme not configurable upstream html_theme = 'default' diff --git a/docs/deprecated-level-10-moban-dependency-as-git-repo/.moban.yml b/docs/deprecated-level-10-moban-dependency-as-git-repo/.moban.yml new file mode 100644 index 00000000..9286746c --- /dev/null +++ b/docs/deprecated-level-10-moban-dependency-as-git-repo/.moban.yml @@ -0,0 +1,11 @@ +requires: + - https://github.com/moremoban/pypi-mobans +configuration: + template_dir: + - "pypi-mobans:templates" + - local + configuration: config.yml + configuration_dir: "pypi-mobans:config" +targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 diff --git a/docs/deprecated-level-10-moban-dependency-as-git-repo/README.rst b/docs/deprecated-level-10-moban-dependency-as-git-repo/README.rst new file mode 100644 index 00000000..f96262aa --- /dev/null +++ b/docs/deprecated-level-10-moban-dependency-as-git-repo/README.rst @@ -0,0 +1,41 @@ +level 10: moban dependency as git repo +================================================================================ + +Since the support to have a pypi package as dependency, the moban pro user will +find it more useful to have git repo so that the changes to static content +could get propagate as it happens using git push and git pull. + +For now, github.com, gitlab.com and bitbucket.com are supported. Pull request +is welcome to add or improve this feature. + + +Here are the sample file:: + + requires: + - https://github.com/moremoban/pypi-mobans + configuration: + template_dir: + - "pypi-mobans:templates" + - local + configuration: config.yml + targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 + +where `requires` lead to a list of pypi packages. And when you refer to it, +as in level-9 section, please use "pypi-mobans:" + + +Alternative syntax when submodule exists +-------------------------------------------------------------------------------- + +The alternative syntax is:: + + requires: + - type: git + url: https://github.com/your-git-url + submodule: true + branch: your_choice_or_default_branch_if_not_specified + reference: your_alternative_reference_but_not_used_together_with_branch + ... + diff --git a/docs/deprecated-level-10-moban-dependency-as-git-repo/config.yml b/docs/deprecated-level-10-moban-dependency-as-git-repo/config.yml new file mode 100644 index 00000000..a4d9f0c9 --- /dev/null +++ b/docs/deprecated-level-10-moban-dependency-as-git-repo/config.yml @@ -0,0 +1,2 @@ +overrides: data.yml +level10: "moban dependency as git repo" diff --git a/docs/deprecated-level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 b/docs/deprecated-level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 new file mode 100644 index 00000000..a21dd010 --- /dev/null +++ b/docs/deprecated-level-10-moban-dependency-as-git-repo/local/demo.txt.jj2 @@ -0,0 +1 @@ +{{name}}: {{level10}} \ No newline at end of file diff --git a/docs/deprecated-level-10-moban-dependency-as-git-repo/local/mytravis.yml b/docs/deprecated-level-10-moban-dependency-as-git-repo/local/mytravis.yml new file mode 100644 index 00000000..dc7bd8c8 --- /dev/null +++ b/docs/deprecated-level-10-moban-dependency-as-git-repo/local/mytravis.yml @@ -0,0 +1,24 @@ +sudo: false +language: python +notifications: + email: false +python: + - pypy-5.3.1 + - 3.7-dev + - 3.6 + - 3.5 + - 3.4 + - 2.7 +before_install: + - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi + - if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then + mv min_requirements.txt requirements.txt ; + fi + - test ! -f rnd_requirements.txt || + pip install --no-deps -r rnd_requirements.txt + - test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt ; + - pip install -r tests/requirements.txt +script: + - make test +after_success: + codecov diff --git a/docs/deprecated-level-9-moban-dependency-as-pypi-package/.moban.yml b/docs/deprecated-level-9-moban-dependency-as-pypi-package/.moban.yml new file mode 100644 index 00000000..63fbe12f --- /dev/null +++ b/docs/deprecated-level-9-moban-dependency-as-pypi-package/.moban.yml @@ -0,0 +1,11 @@ +requires: + - pypi-mobans-pkg +configuration: + template_dir: + - "pypi-mobans-pkg:templates" + - local + configuration: config.yml + configuration_dir: "pypi-mobans-pkg:config" +targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 diff --git a/docs/deprecated-level-9-moban-dependency-as-pypi-package/README.rst b/docs/deprecated-level-9-moban-dependency-as-pypi-package/README.rst new file mode 100644 index 00000000..1366bbc6 --- /dev/null +++ b/docs/deprecated-level-9-moban-dependency-as-pypi-package/README.rst @@ -0,0 +1,49 @@ +level 9: moban dependency as pypi package +================================================================================ + +Why not enable template reuse? Once a template is written somewhere by somebody, +as long as it is good and useful, it is always to reuse it, isn't it? DRY +principle kicks in. + +Now with moban, it is possible to package up your mobans/templates +into a pypi package and distribute it to the world of moban. + + +Here are the sample file:: + + requires: + - pypi-mobans + configuration: + template_dir: + - "pypi-mobans:templates" + configuration: config.yml + targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 + +where `requires` lead to a list of pypi packages. The short syntax is:: + + requires: + - python-package-name + +When you refer to it in configuration section, here is the syntax:: + + configuration: + - template_dir: + - "python-package-name:relative-folder-inside-the-package" + +Note: when you do not have relative directory, please keep semi-colon:: + + configuration: + template_dir: + - "python-package-name:" + +Alternative syntax +-------------------------------------------------------------------------------- + +The alternative syntax is:: + + requires: + - type: pypi + name: pypi-mobans + ... diff --git a/docs/deprecated-level-9-moban-dependency-as-pypi-package/config.yml b/docs/deprecated-level-9-moban-dependency-as-pypi-package/config.yml new file mode 100644 index 00000000..97730934 --- /dev/null +++ b/docs/deprecated-level-9-moban-dependency-as-pypi-package/config.yml @@ -0,0 +1,2 @@ +overrides: data.yml +level9: "moban dependency as pypi package" diff --git a/docs/deprecated-level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 b/docs/deprecated-level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 new file mode 100644 index 00000000..ee50778b --- /dev/null +++ b/docs/deprecated-level-9-moban-dependency-as-pypi-package/local/demo.txt.jj2 @@ -0,0 +1 @@ +{{name}}: {{level9}} \ No newline at end of file diff --git a/docs/engine.png b/docs/engine.png deleted file mode 100644 index 9cc99d43..00000000 Binary files a/docs/engine.png and /dev/null differ diff --git a/docs/engine.uml b/docs/engine.uml index 03902fbd..404e8663 100644 --- a/docs/engine.uml +++ b/docs/engine.uml @@ -3,10 +3,22 @@ package "moban" { [engine factory] [jinja2 engine] +[data loader] +[yaml loader] +[json loader] +[file system layer] } [lml] +package "pyfilesystem2" { +[fs] +[tar] +[zip] +[file] +[s3] +} + package "moban-mako" { [mako engine] } @@ -15,9 +27,30 @@ package "moban-haml" { [haml engine] } +package "moremobans/gitfs2" { +[git repo] +} + +package "moremobans/pypifs" { +[python package] +} + +[fs.dropbox] + + [engine factory] -> [lml] : get all engines [lml] <-- [jinja2 engine] : register [lml] <.. [mako engine] : register [lml] <.. [haml engine] : register - +[lml] <.. [yaml loader] : register +[lml] <.. [json loader] : register +[data loader] -> [lml] : get all loaders +[file system layer] -> [fs] : access templates,config,output +[fs] <.. [git repo] +[fs] <.. [python package] +[fs] <.. [tar] +[fs] <.. [zip] +[fs] <.. [file] +[fs] <.. [s3] +[fs] <.. [fs.dropbox] @enduml diff --git a/docs/extension.rst b/docs/extension.rst index aa4d6a98..1cf2d9d9 100644 --- a/docs/extension.rst +++ b/docs/extension.rst @@ -16,12 +16,64 @@ Jinja2 Filter .. literalinclude:: ../moban/filters/repr.py +split_length +-------------------------------------------------------------------------------- + +It breaks down the given string into a fixed length paragraph. Here is the syntax:: + + {% for line in your_string | split_length(your_line_with) %} + {{line}} + {% endfor %} + +It is used to keep changelog formatted in +`CHANGELOG.rst.jj2 in pypi-mobans project `_ + +github_expand +-------------------------------------------------------------------------------- + +It expands simple hashtags into github issues. Here is the syntax:: + + {{ your_github_string | github_expand }} + + +It makes it easy to mention github reference in change log in all projects. Here is +the place it is applied: +`CHANGELOG.rst.jj2 in pypi-mobans project `_ + + +Here is Grammar in the changelog.yml:: + + =============== ============================== + Syntax Meaning + =============== ============================== + `#1` moban issues 1 + `PR#1` moban pull request 1 + `pyexcel#1` other project issues 1 + `pyexcel#PR#1` other project pulll request 1 + =============== ============================== + +More details can be found in `moban's changelog.yml `_ + +`repr` +-------------------------------------------------------------------------------- + +Returns a single quoted string in the templated file + + +Built-in Tests +================================================================================ Jinja2 Test ******************* .. literalinclude:: ../moban/tests/files.py + +`exists` +-------------------------------------------------------------------------------- + +Test if a file exists or not + Jinja2 Globals ******************* @@ -37,7 +89,7 @@ Template engine extension for Moban moban version 0.2 started using `lml`_ to employ loose couple plugins. Other template engines, such as marko, haml can be plugged into moban seamless. -.. image:: engine.png +.. image:: images/engine.svg In order plugin other template engines, it is to write a lml plugin. The following is an example starting point for any template engine. @@ -48,7 +100,7 @@ is an example starting point for any template engine. constants.TEMPLATE_ENGINE_EXTENSION, tags=["file", "extensions", "for", "your", "template"] ) class Engine(object): - def __init__(self, template_dirs): + def __init__(self, template_fs, options=None): """ A list template directories will be given to your engine class """ diff --git a/docs/images/engine.svg b/docs/images/engine.svg new file mode 100644 index 00000000..97c6babc --- /dev/null +++ b/docs/images/engine.svg @@ -0,0 +1,69 @@ +mobanpyfilesystem2moban-makomoban-hamlmoremobans/gitfs2moremobans/pypifsengine factoryjinja2 enginedata loaderyaml loaderjson loaderfile system layerfstarzipfiles3mako enginehaml enginegit repopython packagelmlfs.dropboxget all enginesregisterregisterregisterregisterregisterget all loadersaccess templates,config,output \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index ace1fb1b..d5a85674 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,7 +34,8 @@ examples folder. level-17-force-template-type-from-moban-file/README.rst level-18-user-defined-template-types/README.rst level-19-moban-a-sub-group-in-targets/README.rst - + level-20-templates-configs-in-zip-or-tar/README.rst + level-21-copy-templates-into-an-alien-file-system/README.rst For more complex use case, please look at `its usage in pyexcel project `_ diff --git a/docs/level-1-jinja2-cli/README.rst b/docs/level-1-jinja2-cli/README.rst index e8d4d673..845cad76 100644 --- a/docs/level-1-jinja2-cli/README.rst +++ b/docs/level-1-jinja2-cli/README.rst @@ -40,3 +40,11 @@ Then go to `docs/level-1-jinja2-cli`. here are different commands to evaluate it moban -t a.template because moban looks for `data.yml` by default + +As well, you can define your own variable: + +.. code-block:: bash + + moban -D hello=maailman -t a.template + +And when you check 'moban.output', you will find you have overwritten data.yaml. diff --git a/docs/level-10-moban-dependency-as-git-repo/.moban.yml b/docs/level-10-moban-dependency-as-git-repo/.moban.yml index 9286746c..fc132401 100644 --- a/docs/level-10-moban-dependency-as-git-repo/.moban.yml +++ b/docs/level-10-moban-dependency-as-git-repo/.moban.yml @@ -1,11 +1,9 @@ -requires: - - https://github.com/moremoban/pypi-mobans configuration: template_dir: - - "pypi-mobans:templates" + - "git://github.com/moremoban/pypi-mobans.git!/templates" - local configuration: config.yml - configuration_dir: "pypi-mobans:config" + configuration_dir: "git://github.com/moremoban/pypi-mobans.git!/config" targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 diff --git a/docs/level-10-moban-dependency-as-git-repo/README.rst b/docs/level-10-moban-dependency-as-git-repo/README.rst index f96262aa..6b1bc912 100644 --- a/docs/level-10-moban-dependency-as-git-repo/README.rst +++ b/docs/level-10-moban-dependency-as-git-repo/README.rst @@ -1,6 +1,10 @@ level 10: moban dependency as git repo ================================================================================ +.. note:: + + You will need to install gitfs2 + Since the support to have a pypi package as dependency, the moban pro user will find it more useful to have git repo so that the changes to static content could get propagate as it happens using git push and git pull. @@ -11,13 +15,12 @@ is welcome to add or improve this feature. Here are the sample file:: - requires: - - https://github.com/moremoban/pypi-mobans configuration: template_dir: - - "pypi-mobans:templates" + - "git://github.com/moremoban/pypi-mobans.git!/templates" - local configuration: config.yml + configuration_dir: "git://github.com/moremoban/pypi-mobans.git!/config" targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 @@ -26,16 +29,21 @@ where `requires` lead to a list of pypi packages. And when you refer to it, as in level-9 section, please use "pypi-mobans:" -Alternative syntax when submodule exists +The syntax when submodule exists -------------------------------------------------------------------------------- -The alternative syntax is:: +The sumodule syntax is:: - requires: - - type: git - url: https://github.com/your-git-url - submodule: true - branch: your_choice_or_default_branch_if_not_specified - reference: your_alternative_reference_but_not_used_together_with_branch - ... + configuration: + template_dir: + - "git://github.com/moremoban/pypi-mobans.git?submodule=true&branch=your_choice_or_default_branch_if_not_specified!/templates" + - local + + +If you have reference instead of branch:: + + configuration: + template_dir: + - "git://github.com/moremoban/pypi-mobans.git?submodule=true&reference=your_alternative_reference_but_not_used_together_with_branch!/templates" + - local diff --git a/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml b/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml new file mode 100644 index 00000000..1caa249e --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/.moban.yml @@ -0,0 +1,13 @@ +configuration: + configuration_dir: 'tar://custom-config.tar' + template_dir: + - zip://templates.zip + - cool-templates + - '.' +targets: + - output: 'tar://a.tar!/a.output' + configuration: data.yml + template: template.in.zip.jj2 + - output: 'zip://a.zip!/a.output2' + configuration: data2.yml + template: subfolder/template.in.zip.jj2 diff --git a/docs/level-20-templates-configs-in-zip-or-tar/README.rst b/docs/level-20-templates-configs-in-zip-or-tar/README.rst new file mode 100644 index 00000000..16977c2e --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/README.rst @@ -0,0 +1,57 @@ +Level 20: templates, files in a zip or tar +================================================================================ + +On top of level 6, you could have files in a zip or tar. +In the following example:: + + configuration: + configuration_dir: 'tar://custom-config.tar' + template_dir: + - zip://templates.zip + - cool-templates + - '.' + targets: + - output: 'tar://a.tar/a.output' + configuration: data.yml + template: template.in.zip.jj2 + - output: 'zip://a.zip/a.output2' + configuration: data2.yml + template: subfolder/template.in.zip.jj2 + +where `template.in.zip.jj2` were loaded from a zip file + + +Evaluation +-------------------------------------------------------------------------------- + +Please go to `docs/level-20-templates-configs-in-zip-or-tar` directory. + +Here is the command to launch it: + +.. code-block:: bash + + moban + +'a.output' is the generated file in a.tar:: + + ========header============ + + world + + shijie + + this demonstrations jinja2's include statement + + ========footer============ + +`a.output2` is in a.zip:: + + ========header============ + + world2 + + shijie + + this demonstrations jinja2's include statement + + ========footer============ diff --git a/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/base.jj2 b/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/base.jj2 new file mode 100644 index 00000000..17b0361b --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/base.jj2 @@ -0,0 +1,11 @@ +{%block header %} +{%endblock%} + +{{hello}} + +{{nihao}} + +{%include 'cool.template.jj2'%} + +{%block footer %} +{%endblock%} diff --git a/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/cool.template.jj2 b/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/cool.template.jj2 new file mode 100644 index 00000000..863bf43e --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/cool-templates/cool.template.jj2 @@ -0,0 +1 @@ +this demonstrates jinja2's include statement diff --git a/docs/level-20-templates-configs-in-zip-or-tar/custom-config.tar b/docs/level-20-templates-configs-in-zip-or-tar/custom-config.tar new file mode 100644 index 00000000..b82f8705 Binary files /dev/null and b/docs/level-20-templates-configs-in-zip-or-tar/custom-config.tar differ diff --git a/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/subfolder/template.in.zip.jj2 b/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/subfolder/template.in.zip.jj2 new file mode 100644 index 00000000..44672595 --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/subfolder/template.in.zip.jj2 @@ -0,0 +1,9 @@ +{%extends 'base.jj2' %} + +{%block header %} +========header============ +{%endblock%} + +{%block footer %} +========footer============ +{%endblock%} diff --git a/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/template.in.zip.jj2 b/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/template.in.zip.jj2 new file mode 100644 index 00000000..44672595 --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/custom-templates/template.in.zip.jj2 @@ -0,0 +1,9 @@ +{%extends 'base.jj2' %} + +{%block header %} +========header============ +{%endblock%} + +{%block footer %} +========footer============ +{%endblock%} diff --git a/docs/level-20-templates-configs-in-zip-or-tar/data.yml b/docs/level-20-templates-configs-in-zip-or-tar/data.yml new file mode 100644 index 00000000..3d225095 --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/data.yml @@ -0,0 +1,2 @@ +overrides: data.in.tar.yaml +hello: world diff --git a/docs/level-20-templates-configs-in-zip-or-tar/data2.yml b/docs/level-20-templates-configs-in-zip-or-tar/data2.yml new file mode 100644 index 00000000..58a044c2 --- /dev/null +++ b/docs/level-20-templates-configs-in-zip-or-tar/data2.yml @@ -0,0 +1,2 @@ +overrides: data.in.tar.yaml +hello: world2 diff --git a/docs/level-20-templates-configs-in-zip-or-tar/templates.zip b/docs/level-20-templates-configs-in-zip-or-tar/templates.zip new file mode 100644 index 00000000..09308e5f Binary files /dev/null and b/docs/level-20-templates-configs-in-zip-or-tar/templates.zip differ diff --git a/docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml b/docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml new file mode 100644 index 00000000..4ae79020 --- /dev/null +++ b/docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml @@ -0,0 +1,19 @@ +configuration: + template_dir: + - "zip://template-sources.zip" +targets: + - output: "zip://my.zip!/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip!/target_without_template_type" + template: file_extension_will_trigger.copy + - "zip://my.zip!/target_in_short_form": as_long_as_this_one_has.copy + - output: "zip://my.zip!/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip!/test-dir" + template: dir-for-copying + template_type: copy + - output: "zip://my.zip!/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy diff --git a/docs/level-21-copy-templates-into-an-alien-file-system/README.rst b/docs/level-21-copy-templates-into-an-alien-file-system/README.rst new file mode 100644 index 00000000..83e20dc8 --- /dev/null +++ b/docs/level-21-copy-templates-into-an-alien-file-system/README.rst @@ -0,0 +1,83 @@ +Level 21: template copying from a zip to a zip +================================================================================ + +In level 15, with `.moban.yml`, you can copy templates to your destination. Now +with similiar moban syntax, let me show how to create a new zip file where +all templates are copied to. + +Explicit syntax:: + + targets: + - output: "zip://your.zip/explicit" + template: template_file + template_type: copy + + +Implicit syntax:: + + targets: + - output: "zip://your.zip/implicit" + template: template_file.copy + + +Shorthand syntax:: + + targets: + - "zip://your.zip/shorthand": template_file.copy + + +No implicit nor short hand syntax for the following directory copying unless +you take a look at `force-template-type`. When you read +`level-17-force-template-type-from-moban-file/README.rst`, you will find +out more. + + +Directory copying syntax:: + + + targets: + - output: "zip://your.zip/dest-dir" + template: source-dir + template_type: copy + + +Recursive directory copying syntax:: + + + targets: + - output: "zip://your.zip/dest-dir" + template: source-dir/** + template_type: copy + + +Evaluation +-------------------------------------------------------------------------------- + +Here is example moban file for copying:: + + configuration: + template_dir: + - "zip://template-sources.zip" + targets: + - output: "zip://my.zip/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip/target_without_template_type" + template: file_extension_will_trigger.copy + - "zip://my.zip/target_in_short_form": as_long_as_this_one_has.copy + - output: "zip://my.zip/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip/test-dir" + template: dir-for-copying + template_type: copy + - output: "zip://my.zip/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy + + +template copy does: + + +#. copies any template inside pre-declared template directory to anywhere. moban will create directory if needed. +#. copies any directory to anywhere. If "**" is followed, moban attempts to do recursive copying. diff --git a/docs/level-21-copy-templates-into-an-alien-file-system/template-sources.zip b/docs/level-21-copy-templates-into-an-alien-file-system/template-sources.zip new file mode 100644 index 00000000..5085110c Binary files /dev/null and b/docs/level-21-copy-templates-into-an-alien-file-system/template-sources.zip differ diff --git a/docs/level-3-data-override/README.rst b/docs/level-3-data-override/README.rst index c163c38a..66109d58 100644 --- a/docs/level-3-data-override/README.rst +++ b/docs/level-3-data-override/README.rst @@ -1,7 +1,8 @@ Level 3: data override ================================================================================ -What `moban` brings on the table is data inheritance by introducing `overrides` key word in the yaml file:: +What `moban` brings on the table is data inheritance by introducing `overrides` +key word in the yaml file:: overrides: data.base.yaml .... @@ -30,3 +31,27 @@ command to launch it: shijie ========footer============ + + +New development +================================================================================ + +Since verison 0.6.0, `overrides` syntax support two more use cases: + +1 override more than one configuration file +--------------------------------------------- + +For example:: + + overrides: + - config-file-a.yaml + - config-file-b.yaml + +2 override more than one configuration file +--------------------------------------------- + +For example:: + + overrides: + - config-file-a.yaml: keya + - config-file-b.yaml: keyb diff --git a/docs/level-9-moban-dependency-as-pypi-package/.moban.yml b/docs/level-9-moban-dependency-as-pypi-package/.moban.yml index 63fbe12f..b6c703cc 100644 --- a/docs/level-9-moban-dependency-as-pypi-package/.moban.yml +++ b/docs/level-9-moban-dependency-as-pypi-package/.moban.yml @@ -1,11 +1,9 @@ -requires: - - pypi-mobans-pkg configuration: template_dir: - - "pypi-mobans-pkg:templates" + - "pypi://pypi-mobans-pkg/resources/templates" - local configuration: config.yml - configuration_dir: "pypi-mobans-pkg:config" + configuration_dir: "pypi://pypi-mobans-pkg/resources/config" targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 diff --git a/docs/level-9-moban-dependency-as-pypi-package/README.rst b/docs/level-9-moban-dependency-as-pypi-package/README.rst index 7bc94496..6d69f871 100644 --- a/docs/level-9-moban-dependency-as-pypi-package/README.rst +++ b/docs/level-9-moban-dependency-as-pypi-package/README.rst @@ -1,6 +1,10 @@ level 9: moban dependency as pypi package ================================================================================ +.. note:: + + You will need to install pypifs + Why not enable template reuse? Once a template is written somewhere by somebody, as long as it is good and useful, it is always to reuse it, isn't it? DRY principle kicks in. @@ -11,39 +15,23 @@ into a pypi package and distribute it to the world of moban. Here are the sample file:: - requires: - - pypi-mobans configuration: template_dir: - - "pypi-mobans:templates" + - "pypi://pypi-mobans-pkg/resources/templates" configuration: config.yml + configuration_dir: "pypi://pypi-mobans-pkg/config" targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 -where `requires` lead to a list of pypi packages. The short syntax is:: - - requires: - - python-package-name - When you refer to it in configuration section, here is the syntax:: configuration: - template_dir: - - "python-package-name:relative-folder-inside-the-package" + - "pypi://python-package-name/relative-folder-inside-the-package" -Note: when you do not have relative directory, please keep semi-colon:: +Note: when you do not have relative directory:: configuration: template_dir: - - "python-package-name:" - -Alternative syntax --------------------------------------------------------------------------------- - -The alternative syntax is:: - - requires: - - type: pypi - name: pypi-mobans - ... + - "pypi://python-package-name" diff --git a/min_requirements.txt b/min_requirements.txt index b5c854ba..c3cac050 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -4,7 +4,7 @@ ruamel.yaml==0.15.5;python_version != '3.4' and python_version < '3.7' ruamel.yaml==0.15.98;python_version == '3.8' jinja2==2.7.1 lml==0.0.9 -appdirs==1.2.0 +appdirs==1.4.3 crayons== 0.1.0 -GitPython==2.0.0 -git-url-parse==1.2.2 +fs==2.4.11 +jinja2-fsloader==0.2.0 diff --git a/moban/_version.py b/moban/_version.py index b407eb3a..6197ada2 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.5.0" +__version__ = "0.6.0" __author__ = "C. W." diff --git a/moban/buffered_writer.py b/moban/buffered_writer.py new file mode 100644 index 00000000..ff980895 --- /dev/null +++ b/moban/buffered_writer.py @@ -0,0 +1,48 @@ +import os +import sys + +import fs +import fs.path +from moban import utils, file_system + +PY2 = sys.version_info[0] == 2 + + +class BufferedWriter(object): + def __init__(self): + self.fs_list = {} + + def write_file_out(self, filename, content): + if file_system.is_zip_alike_url(filename): + self.write_file_out_to_zip(filename, content) + else: + write_file_out(filename, content) + + def write_file_out_to_zip(self, filename, content): + zip_file, file_name = file_system.url_split(filename) + if zip_file not in self.fs_list: + self.fs_list[zip_file] = fs.open_fs( + file_system.to_unicode(zip_file), create=True + ) + base_dirs = fs.path.dirname(file_name) + if not self.fs_list[zip_file].exists(base_dirs): + self.fs_list[zip_file].makedirs(base_dirs) + self.fs_list[zip_file].writebytes( + file_system.to_unicode(file_name), content + ) + + def close(self): + for fsx in self.fs_list.values(): + fsx.close() + + +def write_file_out(filename, content): + if PY2 and content.__class__.__name__ == "unicode": + content = content.encode("utf-8") + + if not file_system.is_zip_alike_url(filename): + dest_folder = os.path.dirname(filename) + if dest_folder: + utils.mkdir_p(dest_folder) + + file_system.write_bytes(filename, content) diff --git a/moban/constants.py b/moban/constants.py index 507ee903..8b86bb64 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -1,4 +1,4 @@ -import os +from moban import file_system # Template type TEMPLATE_JINJA2 = "jinja2" @@ -36,14 +36,21 @@ LABEL_FORCE = "force" LABEL_VERSION = "version" LABEL_GROUP = "group" +LABEL_DEFINE = "define" +CLI_DICT = "user_dict" DEFAULT_CONFIGURATION_DIRNAME = ".%s.cd" % PROGRAM_NAME DEFAULT_TEMPLATE_DIRNAME = ".%s.td" % PROGRAM_NAME DEFAULT_OPTIONS = { # .moban.cd, default configuration dir - LABEL_CONFIG_DIR: os.path.join(".", DEFAULT_CONFIGURATION_DIRNAME), + LABEL_CONFIG_DIR: file_system.path_join( + ".", DEFAULT_CONFIGURATION_DIRNAME + ), # .moban.td, default template dirs - LABEL_TMPL_DIRS: [".", os.path.join(".", DEFAULT_TEMPLATE_DIRNAME)], + LABEL_TMPL_DIRS: [ + ".", + file_system.path_join(".", DEFAULT_TEMPLATE_DIRNAME), + ], # moban.output, default output file name LABEL_OUTPUT: "%s.output" % PROGRAM_NAME, # data.yml, default data input file @@ -65,6 +72,7 @@ LABEL_DEST = "destination" LABEL_FORCE_TEMPLATE_TYPE = "force_template_type" LABEL_TEMPLATE_TYPES = "template_types" +LABEL_VERBOSE = "verbose" # error messages ERROR_DATA_FILE_NOT_FOUND = "Both %s and %s does not exist" @@ -123,3 +131,11 @@ MESSAGE_DEPRECATE_COPY_SINCE_0_4_0 = ( "'%s:' is deprecated since 0.4.0! " + "Please use 'targets:' instead" ) % TEMPLATE_COPY +MESSAGE_DEPRECATE_MOBAN_NOTATION_SINCE_0_6_0 = ( + "File path notation is deprecated since 0.6.0! " + + "Please use pypi:// or git:// notation instead" +) +MESSAGE_DEPRECATE_REQUIRES_SINCE_0_6_0 = ( + "'requires' is deprecated since 0.6.0! " + + "Please use pypi:// or git:// notation instead" +) diff --git a/moban/copy/__init__.py b/moban/copy.py similarity index 71% rename from moban/copy/__init__.py rename to moban/copy.py index 1d197d1d..e617366a 100644 --- a/moban/copy/__init__.py +++ b/moban/copy.py @@ -1,4 +1,4 @@ -from moban import utils, constants +from moban import constants, file_system from lml.plugin import PluginInfo @@ -19,15 +19,13 @@ class ContentForwardEngine(object): templating mechanism. """ - def __init__(self, template_dirs, extensions=None): - self.template_dirs = template_dirs + def __init__(self, template_fs, extensions=None): + self.template_fs = template_fs def get_template(self, template_file): - real_file_name = utils.find_file_in_template_dirs( - template_file, self.template_dirs + return self.template_fs.readbytes( + file_system.to_unicode(template_file) ) - with open(real_file_name, "rb") as file_handle: - return file_handle.read() def get_template_from_string(self, string): return string diff --git a/moban/core/__init__.py b/moban/core/__init__.py new file mode 100644 index 00000000..ea8607c9 --- /dev/null +++ b/moban/core/__init__.py @@ -0,0 +1,3 @@ +from moban.core.moban_factory import MobanFactory + +ENGINES = MobanFactory() diff --git a/moban/plugins/context.py b/moban/core/context.py similarity index 61% rename from moban/plugins/context.py rename to moban/core/context.py index cd96570e..10ce96f0 100644 --- a/moban/plugins/context.py +++ b/moban/core/context.py @@ -1,7 +1,9 @@ import os +import copy -from moban import utils, reporter, exceptions -from moban.data_loaders.manager import load_data +from moban import utils, reporter, constants, exceptions +from moban.program_options import OPTIONS +from moban.data_loaders.manager import merge, load_data class Context(object): @@ -13,14 +15,17 @@ def __init__(self, context_dirs): ) def get_data(self, file_name): + custom_data = copy.deepcopy(OPTIONS[constants.CLI_DICT]) try: data = load_data(self.context_dirs, file_name) - utils.merge(data, self.__cached_environ_variables) - return data + merge(custom_data, data) + merge(custom_data, self.__cached_environ_variables) + return custom_data except (IOError, exceptions.IncorrectDataInput) as exception: # If data file doesn't exist: # 1. Alert the user of their (potential) mistake # 2. Attempt to use environment vars as data reporter.report_warning_message(str(exception)) reporter.report_using_env_vars() - return self.__cached_environ_variables + merge(custom_data, self.__cached_environ_variables) + return custom_data diff --git a/moban/plugins/template.py b/moban/core/moban_factory.py similarity index 71% rename from moban/plugins/template.py rename to moban/core/moban_factory.py index 8a72f9e7..adc4c1b1 100644 --- a/moban/plugins/template.py +++ b/moban/core/moban_factory.py @@ -2,12 +2,14 @@ import sys import logging -from moban import repo, utils, reporter, constants, exceptions +from moban import utils, reporter, constants, exceptions, file_system +from fs.errors import ResourceNotFound from lml.plugin import PluginManager from moban.hashstore import HASH_STORE -from moban.plugins.context import Context -from moban.plugins.library import LIBRARIES -from moban.plugins.strategy import Strategy +from moban.deprecated import deprecated_moban_path_notation +from moban.core.context import Context +from moban.core.strategy import Strategy +from moban.buffered_writer import BufferedWriter log = logging.getLogger(__name__) PY3_ABOVE = sys.version_info[0] > 2 @@ -28,6 +30,10 @@ def register_options(self, template_types): self.options_registry.update(template_types) def get_engine(self, template_type, template_dirs, context_dirs): + template_dirs = list(expand_template_directories(template_dirs)) + template_dirs = utils.verify_the_existence_of_directories( + template_dirs + ) if template_type in self.options_registry: custom_engine_spec = self.options_registry[template_type] engine_cls = self.load_me_now( @@ -38,8 +44,9 @@ def get_engine(self, template_type, template_dirs, context_dirs): engine_cls = self.load_me_now(template_type) engine_extensions = self.extensions.get(template_type) options = dict(extensions=engine_extensions) - engine = engine_cls(template_dirs, options) - return MobanEngine(template_dirs, context_dirs, engine) + template_fs = file_system.get_multi_fs(template_dirs) + engine = engine_cls(template_fs, options) + return MobanEngine(template_fs, context_dirs, engine) def get_primary_key(self, template_type): for key, item in self.options_registry.items(): @@ -56,15 +63,14 @@ def raise_exception(self, key): class MobanEngine(object): - def __init__(self, template_dirs, context_dirs, engine): - template_dirs = list(expand_template_directories(template_dirs)) - utils.verify_the_existence_of_directories(template_dirs) + def __init__(self, template_fs, context_dirs, engine): context_dirs = expand_template_directory(context_dirs) self.context = Context(context_dirs) - self.template_dirs = template_dirs + self.template_fs = template_fs self.engine = engine self.templated_count = 0 self.file_count = 0 + self.buffered_writer = BufferedWriter() def report(self): if self.templated_count == 0: @@ -78,12 +84,15 @@ def number_of_templated_files(self): return self.templated_count def render_to_file(self, template_file, data_file, output_file): - self.file_count = 1 + template_file = file_system.to_unicode(template_file) data = self.context.get_data(data_file) template = self.engine.get_template(template_file) - template_abs_path = utils.get_template_path( - self.template_dirs, template_file - ) + try: + template_abs_path = self.template_fs.geturl( + template_file, purpose="fs" + ) + except ResourceNotFound: + template_abs_path = template_file flag = self.apply_template( template_abs_path, template, data, output_file @@ -91,11 +100,11 @@ def render_to_file(self, template_file, data_file, output_file): if flag: reporter.report_templating(template_file, output_file) self.templated_count += 1 + self.buffered_writer.close() def render_string_to_file( self, template_in_string, data_file, output_file ): - self.file_count = 1 template = self.engine.get_template_from_string(template_in_string) template_abs_path = template_in_string[:10] + "..." data = self.context.get_data(data_file) @@ -105,41 +114,50 @@ def render_string_to_file( if flag: reporter.report_templating(template_abs_path, output_file) self.templated_count += 1 + self.buffered_writer.close() def apply_template(self, template_abs_path, template, data, output_file): rendered_content = self.engine.apply_template( template, data, output_file ) - if PY3_ABOVE: - if not isinstance(rendered_content, bytes): - rendered_content = rendered_content.encode("utf-8") + if not isinstance(rendered_content, bytes): + rendered_content = rendered_content.encode("utf-8") try: flag = HASH_STORE.is_file_changed( output_file, rendered_content, template_abs_path ) if flag: - utils.write_file_out(output_file, rendered_content) - utils.file_permissions_copy(template_abs_path, output_file) + self.buffered_writer.write_file_out( + output_file, rendered_content + ) + if not file_system.is_zip_alike_url(output_file): + file_system.file_permissions_copy( + template_abs_path, output_file + ) return flag except exceptions.FileNotFound: - utils.write_file_out(output_file, rendered_content) + # the template is a string from command line + log.info("{} is not a file".format(template_abs_path)) + self.buffered_writer.write_file_out(output_file, rendered_content) return True def render_to_files(self, array_of_template_targets): sta = Strategy(array_of_template_targets) + sta.process() choice = sta.what_to_do() if choice == Strategy.DATA_FIRST: self._render_with_finding_data_first(sta.data_file_index) else: self._render_with_finding_template_first(sta.template_file_index) + self.buffered_writer.close() def _render_with_finding_template_first(self, template_file_index): for (template_file, data_output_pairs) in template_file_index.items(): template = self.engine.get_template(template_file) - template_abs_path = utils.get_template_path( - self.template_dirs, template_file + template_abs_path = self.template_fs.geturl( + file_system.to_unicode(template_file), purpose="fs" ) for (data_file, output) in data_output_pairs: data = self.context.get_data(data_file) @@ -156,8 +174,8 @@ def _render_with_finding_data_first(self, data_file_index): data = self.context.get_data(data_file) for (template_file, output) in template_output_pairs: template = self.engine.get_template(template_file) - template_abs_path = utils.get_template_path( - self.template_dirs, template_file + template_abs_path = self.template_fs.geturl( + file_system.to_unicode(template_file), purpose="fs" ) flag = self.apply_template( template_abs_path, template, data, output @@ -180,29 +198,12 @@ def expand_template_directories(dirs): def expand_template_directory(directory): log.debug("Expanding %s..." % directory) translated_directory = None - if ":" in directory and directory[1] != ":": - library_or_repo_name, relative_path = directory.split(":") - potential_repo_path = os.path.join( - repo.get_moban_home(), library_or_repo_name - ) - if os.path.exists(potential_repo_path): - # expand repo template path - if relative_path: - translated_directory = os.path.join( - potential_repo_path, relative_path - ) - else: - translated_directory = potential_repo_path - else: - # expand pypi template path - library_path = LIBRARIES.resource_path_of(library_or_repo_name) - if relative_path: - translated_directory = os.path.join( - library_path, relative_path - ) - else: - translated_directory = library_path + if ":" in directory and directory[1] != ":" and "://" not in directory: + translated_directory = deprecated_moban_path_notation(directory) + elif "://" in directory: + translated_directory = directory else: # local template path - translated_directory = os.path.abspath(directory) + translated_directory = os.path.normcase(os.path.abspath(directory)) + translated_directory = file_system.fs_url(translated_directory) return translated_directory diff --git a/moban/plugins/strategy.py b/moban/core/strategy.py similarity index 100% rename from moban/plugins/strategy.py rename to moban/core/strategy.py diff --git a/moban/data_loaders/json_loader.py b/moban/data_loaders/json_loader.py index b5726fee..6393af47 100644 --- a/moban/data_loaders/json_loader.py +++ b/moban/data_loaders/json_loader.py @@ -2,6 +2,7 @@ from moban import constants from lml.plugin import PluginInfo +from moban.file_system import open_file @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["json"]) @@ -9,6 +10,6 @@ def open_json(file_name): """ returns json contents as string """ - with open(file_name, "r") as json_data: - data = json.load(json_data) + with open_file(file_name) as json_file: + data = json.load(json_file) return data diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index 3dd402a9..6c7a9097 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -1,6 +1,8 @@ -import os +from collections import OrderedDict -from moban import utils, constants +import moban.data_loaders.yaml # noqa: F401 +import moban.data_loaders.json_loader # noqa: F401 +from moban import constants, file_system from lml.plugin import PluginManager @@ -9,7 +11,7 @@ def __init__(self): super(AnyDataLoader, self).__init__(constants.DATA_LOADER_EXTENSION) def get_data(self, file_name): - file_extension = os.path.splitext(file_name)[1] + file_extension = file_system.path_splitext(file_name)[1] file_type = file_extension if file_extension.startswith("."): file_type = file_type[1:] @@ -25,17 +27,62 @@ def get_data(self, file_name): def load_data(base_dir, file_name): - abs_file_path = utils.search_file(base_dir, file_name) + abs_file_path = search_file(base_dir, file_name) data = LOADER.get_data(abs_file_path) if data is not None: - parent_data = None + parent_data = OrderedDict() if base_dir and constants.LABEL_OVERRIDES in data: - parent_data = load_data( - base_dir, data.pop(constants.LABEL_OVERRIDES) - ) + overrides = data.pop(constants.LABEL_OVERRIDES) + if not isinstance(overrides, list): + overrides = [overrides] + for parent_file in overrides: + file_name, key = parent_file, None + if ":" in parent_file: + file_name, key = parent_file.split(":") + child_data = load_data(base_dir, file_name) + if data: + if key: + child_data = OrderedDict({key: child_data[key]}) + parent_data = merge(parent_data, child_data) if parent_data: - return utils.merge(data, parent_data) + return merge(data, parent_data) else: return data else: return None + + +def merge(left, right): + """ + deep merge dictionary on the left with the one + on the right. + + Fill in left dictionary with right one where + the value of the key from the right one in + the left one is missing or None. + """ + if isinstance(left, dict) and isinstance(right, dict): + for key, value in right.items(): + if key not in left: + left[key] = value + elif left[key] is None: + left[key] = value + else: + left[key] = merge(left[key], value) + return left + + +def search_file(base_dir, file_name): + the_file = file_name + if not file_system.exists(the_file): + if base_dir: + file_under_base_dir = file_system.url_join(base_dir, the_file) + if file_system.exists(file_under_base_dir): + the_file = file_system.fs_url(file_under_base_dir) + else: + raise IOError( + constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) + ) + else: + raise IOError(constants.ERROR_DATA_FILE_ABSENT % the_file) + return the_file diff --git a/moban/data_loaders/yaml.py b/moban/data_loaders/yaml.py index 8fce4710..f082730c 100644 --- a/moban/data_loaders/yaml.py +++ b/moban/data_loaders/yaml.py @@ -1,11 +1,12 @@ from moban import constants from lml.plugin import PluginInfo from ruamel.yaml import YAML +from moban.file_system import open_file @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["yaml", "yml"]) def open_yaml(file_name): - with open(file_name, "r") as data_yaml: + with open_file(file_name) as data_yaml: yaml = YAML(typ="rt") data = yaml.load(data_yaml) return data diff --git a/moban/definitions.py b/moban/definitions.py index 8ae4ea45..982e3e5d 100644 --- a/moban/definitions.py +++ b/moban/definitions.py @@ -1,40 +1,6 @@ -import os - from moban import constants -class GitRequire(object): - def __init__( - self, git_url=None, branch=None, submodule=False, reference=None - ): - self.git_url = git_url - self.submodule = submodule - self.branch = branch - self.reference = reference - - def clone_params(self): - clone_params = { - "single_branch": True, - "depth": constants.DEFAULT_CLONE_DEPTH, - } - if self.branch is not None: - clone_params["branch"] = self.branch - elif self.reference is not None: - clone_params["reference"] = self.reference - return clone_params - - def __eq__(self, other): - return ( - self.git_url == other.git_url - and self.submodule == other.submodule - and self.branch == other.branch - and self.reference == other.reference - ) - - def __repr__(self): - return "%s,%s,%s" % (self.git_url, self.branch, self.submodule) - - class TemplateTarget(object): def __init__( self, @@ -54,7 +20,9 @@ def __init__( def set_template_type(self, new_template_type): self.template_type = new_template_type if self.original_output.endswith(self.template_type): - self.output, _ = os.path.splitext(self.original_output) + self.output = self.original_output.replace( + "." + self.template_type, "" + ) else: self.output = self.original_output diff --git a/moban/deprecated.py b/moban/deprecated.py deleted file mode 100644 index 4c8c03a6..00000000 --- a/moban/deprecated.py +++ /dev/null @@ -1,15 +0,0 @@ -from functools import wraps - -from moban import reporter - - -def deprecated(message): - def tags_decorator(func): - @wraps(func) - def func_wrapper(*args, **kwds): - reporter.report_warning_message(message) - return func(*args, **kwds) - - return func_wrapper - - return tags_decorator diff --git a/moban/deprecated/__init__.py b/moban/deprecated/__init__.py new file mode 100644 index 00000000..a2218fbb --- /dev/null +++ b/moban/deprecated/__init__.py @@ -0,0 +1,149 @@ +import sys +from functools import wraps + +from moban import plugins, reporter, constants, file_system +from moban.deprecated import repo +from moban.deprecated.repo import git_clone +from moban.deprecated.library import LIBRARIES + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +KNOWN_DOMAIN_FOR_GIT = ["github.com", "gitlab.com", "bitbucket.com"] + + +def deprecated(message): + def tags_decorator(func): + @wraps(func) + def func_wrapper(*args, **kwds): + reporter.report_warning_message(message) + return func(*args, **kwds) + + return func_wrapper + + return tags_decorator + + +class GitRequire(object): + def __init__( + self, git_url=None, branch=None, submodule=False, reference=None + ): + self.git_url = git_url + self.submodule = submodule + self.branch = branch + self.reference = reference + + def clone_params(self): + clone_params = { + "single_branch": True, + "depth": constants.DEFAULT_CLONE_DEPTH, + } + if self.branch is not None: + clone_params["branch"] = self.branch + elif self.reference is not None: + clone_params["reference"] = self.reference + return clone_params + + def __eq__(self, other): + return ( + self.git_url == other.git_url + and self.submodule == other.submodule + and self.branch == other.branch + and self.reference == other.reference + ) + + def __repr__(self): + return "%s,%s,%s" % (self.git_url, self.branch, self.submodule) + + +def pip_install(packages): + import subprocess + + subprocess.check_call( + [sys.executable, "-m", "pip", "install", " ".join(packages)] + ) + + +@deprecated(constants.MESSAGE_DEPRECATE_MOBAN_NOTATION_SINCE_0_6_0) +def deprecated_moban_path_notation(directory): + translated_directory = None + library_or_repo_name, relative_path = directory.split(":") + potential_repo_path = file_system.path_join( + repo.get_moban_home(), library_or_repo_name + ) + if file_system.exists(potential_repo_path): + # expand repo template path + if relative_path: + translated_directory = file_system.path_join( + potential_repo_path, relative_path + ) + else: + translated_directory = potential_repo_path + else: + # expand pypi template path + library_path = LIBRARIES.resource_path_of(library_or_repo_name) + if relative_path: + translated_directory = file_system.path_join( + library_path, relative_path + ) + else: + translated_directory = library_path + return translated_directory + + +@deprecated(constants.MESSAGE_DEPRECATE_REQUIRES_SINCE_0_6_0) +def handle_requires(requires): + pypi_pkgs = [] + git_repos = [] + for require in requires: + if isinstance(require, dict): + require_type = require.get(constants.REQUIRE_TYPE, "") + if require_type.upper() == constants.GIT_REQUIRE: + git_require = GitRequire( + git_url=require.get(constants.GIT_URL), + branch=require.get(constants.GIT_BRANCH), + reference=require.get(constants.GIT_REFERENCE), + submodule=require.get(constants.GIT_HAS_SUBMODULE, False), + ) + + git_repos.append(git_require) + elif require_type.upper() == constants.PYPI_REQUIRE: + pypi_pkgs.append(require.get(constants.PYPI_PACKAGE_NAME)) + else: + if is_repo(require): + git_repos.append(GitRequire(require)) + else: + pypi_pkgs.append(require) + if pypi_pkgs: + pip_install(pypi_pkgs) + plugins.make_sure_all_pkg_are_loaded() + if git_repos: + git_clone(git_repos) + + +def is_repo(require): + result = urlparse(require) + return result.scheme != "" and result.netloc in KNOWN_DOMAIN_FOR_GIT + + +@deprecated(constants.MESSAGE_DEPRECATE_COPY_SINCE_0_4_0) +def handle_copy(merged_options, copy_config): + copy_targets = [] + for (dest, src) in _iterate_list_of_dicts(copy_config): + copy_targets.append( + { + constants.LABEL_TEMPLATE: src, + constants.LABEL_OUTPUT: dest, + constants.LABEL_TEMPLATE_TYPE: constants.TEMPLATE_COPY, + } + ) + return copy_targets + + +def _iterate_list_of_dicts(list_of_dict): + for adict in list_of_dict: + for key, value in adict.items(): + yield (key, value) diff --git a/moban/plugins/library.py b/moban/deprecated/library.py similarity index 100% rename from moban/plugins/library.py rename to moban/deprecated/library.py diff --git a/moban/repo.py b/moban/deprecated/repo.py similarity index 88% rename from moban/repo.py rename to moban/deprecated/repo.py index 02fdd6de..47c40635 100644 --- a/moban/repo.py +++ b/moban/deprecated/repo.py @@ -1,8 +1,7 @@ -import os import sys import subprocess -from moban import reporter, constants, exceptions +from moban import reporter, constants, exceptions, file_system from moban.utils import mkdir_p @@ -20,8 +19,8 @@ def git_clone(requires): for require in requires: repo_name = get_repo_name(require.git_url) - local_repo_folder = os.path.join(moban_home, repo_name) - if os.path.exists(local_repo_folder): + local_repo_folder = file_system.path_join(moban_home, repo_name) + if file_system.exists(local_repo_folder): reporter.report_git_pull(repo_name) repo = Repo(local_repo_folder) repo.git.pull() @@ -60,7 +59,7 @@ def get_moban_home(): from appdirs import user_cache_dir home_dir = user_cache_dir(appname=constants.PROGRAM_NAME) - return os.path.join(home_dir, constants.MOBAN_REPOS_DIR_NAME) + return file_system.path_join(home_dir, constants.MOBAN_REPOS_DIR_NAME) def make_sure_git_is_available(): diff --git a/moban/file_system.py b/moban/file_system.py new file mode 100644 index 00000000..f08554f4 --- /dev/null +++ b/moban/file_system.py @@ -0,0 +1,262 @@ +import os +import sys +import stat +import logging +from contextlib import contextmanager + +import fs +import fs.path +from moban import exceptions +from fs.multifs import MultiFS + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +PY2 = sys.version_info[0] == 2 +LOG = logging.getLogger(__name__) + +path_join = fs.path.join +path_splitext = fs.path.splitext + + +def url_join(path, path2): + result = urlparse(path) + if result.scheme and path.endswith(result.scheme): + return path + to_unicode("!/") + path2 + else: + return path + to_unicode("/") + path2 + + +def log_fs_failure(function_in_this_module): + def wrapper(*args, **kwds): + try: + return function_in_this_module(*args, **kwds) + except fs.errors.CreateFailed: + from moban import reporter + + message = "Failed to open %s" % args[0] + LOG.debug(message) + reporter.report_error_message(message) + raise exceptions.FileNotFound(args[0]) + + return wrapper + + +@log_fs_failure +@contextmanager +def open_fs(path): + path = to_unicode(path) + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + the_fs = fs.open_fs(zip_file) + else: + the_fs = fs.open_fs(path) + try: + yield the_fs + finally: + the_fs.close() + + +@log_fs_failure +@contextmanager +def open_file(path): + path = to_unicode(path) + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + the_fs = fs.open_fs(zip_file) + f = the_fs.open(folder) + else: + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + the_fs = fs.open_fs(dir_name) + f = the_fs.open(the_file_name) + try: + yield f + finally: + f.close() + the_fs.close() + + +@log_fs_failure +@contextmanager +def open_binary_file(path): + path = to_unicode(path) + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + the_fs = fs.open_fs(zip_file) + f = the_fs.openbin(folder) + else: + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + the_fs = fs.open_fs(dir_name) + f = the_fs.openbin(the_file_name) + try: + yield f + finally: + f.close() + the_fs.close() + + +@log_fs_failure +def read_unicode(path): + with open_file(path) as file_handle: + return file_handle.read() + + +@log_fs_failure +def read_bytes(path): + with open_binary_file(path) as file_handle: + return file_handle.read() + + +read_binary = read_bytes +read_text = read_unicode + + +@log_fs_failure +def write_bytes(filename, bytes_content): + filename = to_unicode(filename) + if "://" in filename: + zip_file, folder = url_split(filename) + with fs.open_fs(zip_file, create=True) as the_fs: + the_fs.writebytes(folder, bytes_content) + else: + dir_name = fs.path.dirname(filename) + the_file_name = fs.path.basename(filename) + with fs.open_fs(dir_name) as the_fs: + the_fs.writebytes(the_file_name, bytes_content) + + +@log_fs_failure +def is_dir(path): + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + return the_fs.isdir(path) + + +@log_fs_failure +def is_file(path): + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + return the_fs.isfile(path) + + +@log_fs_failure +def exists(path): + path = to_unicode(path) + + if is_zip_alike_url(path): + zip_file, folder = url_split(path) + try: + with fs.open_fs(zip_file) as the_fs: + if folder: + return the_fs.exists(folder) + return True + except fs.errors.CreateFailed: + return False + dir_name = fs.path.dirname(path) + the_file_name = fs.path.basename(path) + + try: + with fs.open_fs(dir_name) as the_fs: + return the_fs.exists(the_file_name) + except fs.errors.CreateFailed: + return False + + +@log_fs_failure +def list_dir(path): + path = to_unicode(path) + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + for file_name in the_fs.listdir(path): + yield file_name + + +@log_fs_failure +def abspath(path): + path = to_unicode(path) + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + return the_fs.getsyspath(path) + + +@log_fs_failure +def fs_url(path): + path = to_unicode(path) + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + return the_fs.geturl(path, purpose="fs") + + +@log_fs_failure +def system_path(path): + path = to_unicode(path) + folder_or_file, path = _path_split(path) + with fs.open_fs(folder_or_file) as the_fs: + return the_fs.getsyspath(path) + + +def to_unicode(path): + if PY2 and path.__class__.__name__ != "unicode": + return u"".__class__(path) + return path + + +def is_zip_alike_url(url): + specs = ["zip://", "tar://"] + for prefix in specs: + if url.startswith(prefix): + return True + else: + return False + + +def file_permissions_copy(source, dest): + source_permissions = file_permissions(source) + dest_permissions = file_permissions(dest) + + if source_permissions != dest_permissions: + os.chmod(dest, source_permissions) + + +def file_permissions(url): + if not exists(url): + raise exceptions.FileNotFound(url) + if sys.platform == "win32": + return 0 + elif is_zip_alike_url(url): + return 755 + else: + unix_path = system_path(url) + return stat.S_IMODE(os.stat(unix_path).st_mode) + + +def url_split(url): + result = urlparse(url) + + if url.endswith(result.scheme): + url_to_file = url + path = None + else: + url_to_file, path = url.split("!/") + + return url_to_file, path + + +def _path_split(url_or_path): + url_or_path = to_unicode(url_or_path) + if is_zip_alike_url(url_or_path): + return url_split(url_or_path) + else: + return fs.path.dirname(url_or_path), fs.path.basename(url_or_path) + + +def get_multi_fs(directories): + filesystem = MultiFS() + for directory in directories: + filesystem.add_fs(directory, fs.open_fs(directory)) + return filesystem diff --git a/moban/hashstore.py b/moban/hashstore.py index 25b72518..636e7165 100644 --- a/moban/hashstore.py +++ b/moban/hashstore.py @@ -1,10 +1,8 @@ -import os import sys import json import hashlib -import moban.utils as utils -import moban.constants as constants +from moban import constants, file_system PY2 = sys.version_info[0] == 2 @@ -13,9 +11,14 @@ class HashStore: IGNORE_CACHE_FILE = False def __init__(self): - self.cache_file = constants.DEFAULT_MOBAN_CACHE_FILE - if os.path.exists(self.cache_file) and self.IGNORE_CACHE_FILE is False: - with open(self.cache_file, "r") as f: + self.cache_file = file_system.to_unicode( + constants.DEFAULT_MOBAN_CACHE_FILE + ) + if ( + file_system.exists(self.cache_file) + and self.IGNORE_CACHE_FILE is False + ): + with file_system.open_file(self.cache_file) as f: self.hashes = json.load(f) else: self.hashes = {} @@ -34,10 +37,10 @@ def is_file_changed(self, file_name, file_content, source_template): def _is_source_updated(self, file_name, file_content, source_template): changed = True content = _mix( - file_content, oct(utils.file_permissions(source_template)) + file_content, oct(file_system.file_permissions(source_template)) ) content_hash = get_hash(content) - if os.path.exists(file_name): + if file_system.exists(file_name): if file_name in self.hashes: if content_hash == self.hashes[file_name]: changed = False @@ -57,9 +60,8 @@ def save_hashes(self): def get_file_hash(afile): - with open(afile, "rb") as handle: - content = handle.read() - content = _mix(content, oct(utils.file_permissions(afile))) + content = file_system.read_bytes(afile) + content = _mix(content, oct(file_system.file_permissions(afile))) return get_hash(content) diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index e80cd15f..eadf6df3 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -1,12 +1,15 @@ import re +import logging from importlib import import_module -from moban import constants, exceptions -from jinja2 import Template, Environment, FileSystemLoader +from moban import constants, file_system +from jinja2 import Template, Environment from lml.loader import scan_plugins_regex from lml.plugin import PluginInfo, PluginManager from jinja2.exceptions import TemplateNotFound +from jinja2_fsloader import FSLoader + JINJA2_LIBRARIES = "^moban_jinja2_.+$" JINJA2_EXENSIONS = [ "moban.jinja2.filters.repr", @@ -15,6 +18,7 @@ "moban.jinja2.tests.files", ] JINJA2_THIRD_PARTY_EXTENSIONS = ["jinja2.ext.do", "jinja2.ext.loopcontrols"] +LOG = logging.getLogger(__name__) class PluginMixin: @@ -53,18 +57,19 @@ def __init__(self): constants.TEMPLATE_ENGINE_EXTENSION, tags=["jinja2", "jinja", "jj2", "j2"] ) class Engine(object): - def __init__(self, template_dirs, options=None): + def __init__(self, template_fs, options=None): """ Contruct a jinja2 template engine - A list template directories will be given to your engine class + an instance of fs.multifs.MultiFS will be given and straightaway it is given + to jinja2.FSLoader. - :param list temp_dirs: a list of template directories + :param fs.multifs.MultiFS template_fs: a MultiFS instance or a FS instance :param dict options: a dictionary containing environmental parameters """ + LOG.debug("Jinja template engine started") load_jinja2_extensions() - self.template_dirs = template_dirs - template_loader = FileSystemLoader(template_dirs) + template_loader = FSLoader(template_fs) env_params = dict( loader=template_loader, keep_trailing_newline=True, @@ -74,6 +79,7 @@ def __init__(self, template_dirs, options=None): extension for extension in JINJA2_THIRD_PARTY_EXTENSIONS ], # get a copy of this global variable ) + self.template_loader = template_loader if options: if "extensions" in options: extensions = options.pop("extensions") @@ -112,7 +118,8 @@ def get_template(self, template_file): try: template = self.jj2_environment.get_template(template_file) except TemplateNotFound: - raise exceptions.FileNotFound("%s does not exist" % template_file) + content = file_system.read_unicode(template_file) + return Template(content) return template def get_template_from_string(self, string): diff --git a/moban/main.py b/moban/main.py index 619e0a61..d7872b1b 100644 --- a/moban/main.py +++ b/moban/main.py @@ -9,13 +9,26 @@ """ import sys +import logging import argparse +import logging.config -from moban import plugins, reporter, constants, mobanfile, exceptions -from moban.utils import merge +from moban import ( + core, + plugins, + reporter, + constants, + mobanfile, + exceptions, + file_system, +) from moban._version import __version__ from moban.hashstore import HASH_STORE -from moban.data_loaders.manager import load_data +from moban.program_options import OPTIONS +from moban.data_loaders.manager import merge, load_data + +LOG = logging.getLogger() +LOG_LEVEL = [logging.WARNING, logging.INFO, logging.DEBUG] def main(): @@ -25,10 +38,16 @@ def main(): parser = create_parser() options = vars(parser.parse_args()) HASH_STORE.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE] + options[constants.CLI_DICT] = handle_custom_variables( + options.pop(constants.LABEL_DEFINE) + ) + OPTIONS.update(options) + handle_verbose(options[constants.LABEL_VERBOSE]) + moban_file = options[constants.LABEL_MOBANFILE] load_engine_factory_and_engines() # Error: jinja2 if removed if moban_file is None: - moban_file = mobanfile.find_default_moban_file() + moban_file = find_default_moban_file() if moban_file: try: count = handle_moban_file(moban_file, options) @@ -38,6 +57,7 @@ def main(): exceptions.NoThirdPartyEngine, exceptions.MobanfileGrammarException, ) as e: + LOG.exception(e) reporter.report_error_message(str(e)) moban_exit(options[constants.LABEL_EXIT_CODE], constants.ERROR) else: @@ -117,11 +137,24 @@ def create_parser(): help="string templates", ) parser.add_argument( - "-v", + "-V", "--%s" % constants.LABEL_VERSION, action="version", version="%(prog)s {v}".format(v=__version__), ) + parser.add_argument( + "-v", + action="count", + dest=constants.LABEL_VERBOSE, + default=0, + help="show verbose", + ) + parser.add_argument( + "-d", + "--%s" % constants.LABEL_DEFINE, + nargs="+", + help="to take a list of VAR=VALUEs", + ) return parser @@ -181,7 +214,7 @@ def handle_command_line(options): act upon command options """ options = merge(options, constants.DEFAULT_OPTIONS) - engine = plugins.ENGINES.get_engine( + engine = core.ENGINES.get_engine( options[constants.LABEL_TEMPLATE_TYPE], options[constants.LABEL_TMPL_DIRS], options[constants.LABEL_CONFIG_DIR], @@ -209,5 +242,34 @@ def handle_command_line(options): return exit_code +def find_default_moban_file(): + for moban_file in constants.DEFAULT_MOBAN_FILES: + if file_system.exists(moban_file): + break + else: + moban_file = None + return moban_file + + def load_engine_factory_and_engines(): plugins.make_sure_all_pkg_are_loaded() + + +def handle_custom_variables(list_of_definitions): + custom_data = {} + if list_of_definitions: + for definition in list_of_definitions: + key, value = definition.split("=") + custom_data[key] = value + + return custom_data + + +def handle_verbose(verbose_level): + if verbose_level > len(LOG_LEVEL): + verbose_level = 3 + level = LOG_LEVEL[verbose_level] + logging.basicConfig( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + level=level, + ) diff --git a/moban/mobanfile/__init__.py b/moban/mobanfile/__init__.py index 2d3dca4e..f3ae753d 100644 --- a/moban/mobanfile/__init__.py +++ b/moban/mobanfile/__init__.py @@ -1,36 +1,26 @@ import os import re import sys +import logging from collections import OrderedDict -from moban import plugins, reporter, constants +from moban import core, reporter, constants from lml.utils import do_import -from moban.repo import git_clone -from moban.utils import merge, pip_install -from moban.deprecated import deprecated -from moban.definitions import GitRequire -from moban.plugins.template import expand_template_directories -from moban.mobanfile.targets import parse_targets, extract_group_targets +from moban.utils import verify_the_existence_of_directories +from moban.deprecated import handle_copy, handle_requires +from moban.mobanfile.targets import ( + parse_targets, + extract_target, + extract_group_targets, +) +from moban.core.moban_factory import expand_template_directories +from moban.data_loaders.manager import merge -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - - -KNOWN_DOMAIN_FOR_GIT = ["github.com", "gitlab.com", "bitbucket.com"] - - -def find_default_moban_file(): - for moban_file in constants.DEFAULT_MOBAN_FILES: - if os.path.exists(moban_file): - break - else: - moban_file = None - return moban_file +LOG = logging.getLogger(__name__) def handle_moban_file_v1(moban_file_configurations, command_line_options): + LOG.info("handling moban file") merged_options = None targets = moban_file_configurations.get(constants.LABEL_TARGETS, []) @@ -56,22 +46,30 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): if plugins_dirs: handle_plugin_dirs(plugins_dirs) + # deprecated requires = moban_file_configurations.get(constants.LABEL_REQUIRES) if requires: handle_requires(requires) # call expand template directory always after handle require please # the penalty is: newly clone repos are not visible - merged_options[constants.LABEL_TMPL_DIRS] = list( - expand_template_directories(merged_options[constants.LABEL_TMPL_DIRS]) + # one more note: verify_the_existence_of_directories will remove non-exist dirs + merged_options[ + constants.LABEL_TMPL_DIRS + ] = verify_the_existence_of_directories( + list( + expand_template_directories( + merged_options[constants.LABEL_TMPL_DIRS] + ) + ) ) extensions = moban_file_configurations.get(constants.LABEL_EXTENSIONS) if extensions: - plugins.ENGINES.register_extensions(extensions) + core.ENGINES.register_extensions(extensions) template_types = merged_options.get(constants.LABEL_TEMPLATE_TYPES) if template_types: - plugins.ENGINES.register_options(template_types) + core.ENGINES.register_options(template_types) if cli_target: number_of_templated_files = handle_targets( @@ -87,27 +85,8 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): return exit_code -@deprecated(constants.MESSAGE_DEPRECATE_COPY_SINCE_0_4_0) -def handle_copy(merged_options, copy_config): - copy_targets = [] - for (dest, src) in _iterate_list_of_dicts(copy_config): - copy_targets.append( - { - constants.LABEL_TEMPLATE: src, - constants.LABEL_OUTPUT: dest, - constants.LABEL_TEMPLATE_TYPE: constants.TEMPLATE_COPY, - } - ) - return copy_targets - - -def _iterate_list_of_dicts(list_of_dict): - for adict in list_of_dict: - for key, value in adict.items(): - yield (key, value) - - def handle_targets(merged_options, targets): + LOG.info("handling targets") list_of_templating_parameters = parse_targets(merged_options, targets) jobs_for_each_engine = OrderedDict() @@ -119,7 +98,7 @@ def handle_targets(merged_options, targets): target.set_template_type(forced_template_type) template_type = target.template_type - primary_template_type = plugins.ENGINES.get_primary_key(template_type) + primary_template_type = core.ENGINES.get_primary_key(template_type) if primary_template_type is None: primary_template_type = merged_options[ constants.LABEL_TEMPLATE_TYPE @@ -133,7 +112,7 @@ def handle_targets(merged_options, targets): count = 0 for template_type in jobs_for_each_engine.keys(): - engine = plugins.ENGINES.get_engine( + engine = core.ENGINES.get_engine( template_type, merged_options[constants.LABEL_TMPL_DIRS], merged_options[constants.LABEL_CONFIG_DIR], @@ -145,8 +124,11 @@ def handle_targets(merged_options, targets): def handle_plugin_dirs(plugin_dirs): + LOG.info("handling plugin dirs {}".format(",".join(plugin_dirs))) for plugin_dir in plugin_dirs: - plugin_path = os.path.dirname(os.path.abspath(plugin_dir)) + plugin_path = os.path.normcase( + os.path.dirname(os.path.abspath(plugin_dir)) + ) if plugin_path not in sys.path: sys.path.append(plugin_path) pysearchre = re.compile(".py$", re.IGNORECASE) @@ -155,61 +137,3 @@ def handle_plugin_dirs(plugin_dirs): for plugin in plugins: plugin_module = os.path.basename(plugin_dir) + "." + plugin do_import(plugin_module) - - -def extract_target(options): - template = options.get(constants.LABEL_TEMPLATE) - config = options.get(constants.LABEL_CONFIG) - output = options.get(constants.LABEL_OUTPUT) - result = [] - if template: - if output is None: - raise Exception( - "Please specify a output file name for %s." % template - ) - if config: - result = { - constants.LABEL_TEMPLATE: template, - constants.LABEL_CONFIG: config, - constants.LABEL_OUTPUT: output, - } - - else: - result = {output: template} - return result - - -def handle_requires(requires): - pypi_pkgs = [] - git_repos = [] - for require in requires: - if isinstance(require, dict): - require_type = require.get(constants.REQUIRE_TYPE, "") - if require_type.upper() == constants.GIT_REQUIRE: - git_repos.append( - GitRequire( - git_url=require.get(constants.GIT_URL), - branch=require.get(constants.GIT_BRANCH), - reference=require.get(constants.GIT_REFERENCE), - submodule=require.get( - constants.GIT_HAS_SUBMODULE, False - ), - ) - ) - elif require_type.upper() == constants.PYPI_REQUIRE: - pypi_pkgs.append(require.get(constants.PYPI_PACKAGE_NAME)) - else: - if is_repo(require): - git_repos.append(GitRequire(require)) - else: - pypi_pkgs.append(require) - if pypi_pkgs: - pip_install(pypi_pkgs) - plugins.make_sure_all_pkg_are_loaded() - if git_repos: - git_clone(git_repos) - - -def is_repo(require): - result = urlparse(require) - return result.scheme != "" and result.netloc in KNOWN_DOMAIN_FOR_GIT diff --git a/moban/mobanfile/targets.py b/moban/mobanfile/targets.py index e32c8767..161de814 100644 --- a/moban/mobanfile/targets.py +++ b/moban/mobanfile/targets.py @@ -1,9 +1,36 @@ import uuid +import logging -from moban import plugins, reporter, constants, exceptions +from moban import core, reporter, constants, exceptions from moban.definitions import TemplateTarget from moban.mobanfile.templates import handle_template +LOG = logging.getLogger(__name__) + + +def extract_target(options): + template, config, output = ( + options.get(constants.LABEL_TEMPLATE), + options.get(constants.LABEL_CONFIG), + options.get(constants.LABEL_OUTPUT), + ) + result = [] + if template: + if output is None: + raise Exception( + "Please specify a output file name for %s." % template + ) + if config: + result = { + constants.LABEL_TEMPLATE: template, + constants.LABEL_CONFIG: config, + constants.LABEL_OUTPUT: output, + } + + else: + result = {output: template} + return result + def extract_group_targets(group, targets): for target in targets: @@ -18,6 +45,7 @@ def extract_group_targets(group, targets): def parse_targets(options, targets): + LOG.info("paring targets..") for target in targets: if constants.LABEL_OUTPUT in target: for template_target in _handle_explicit_target(options, target): @@ -65,7 +93,7 @@ def _handle_explicit_target(options, target): constants.TEMPLATE_TYPES_OPTIONS: template_types_options, } } - plugins.ENGINES.register_options(the_adhoc_type) + core.ENGINES.register_options(the_adhoc_type) template_type = file_extension for src, dest, t_type in handle_template( template_file, output, options[constants.LABEL_TMPL_DIRS] diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 795b15cd..4042ac0e 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -1,76 +1,70 @@ -import os import logging -from moban import reporter -from moban.utils import find_file_in_template_dirs +from moban import reporter, file_system -log = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) def handle_template(template_file, output, template_dirs): - log.info("handling %s" % template_file) - template_file_on_disk = find_file_in_template_dirs( - template_file, template_dirs - ) - if template_file_on_disk is None: - if template_file.endswith("**"): - source_dir = template_file[:-3] - src_path = find_file_in_template_dirs(source_dir, template_dirs) - if src_path: - for a_triple in _listing_directory_files_recusively( - source_dir, src_path, output - ): - yield a_triple - else: - reporter.report_error_message( - "{0} cannot be found".format(template_file) - ) + LOG.info("handling %s" % template_file) + + template_file = file_system.to_unicode(template_file) + multi_fs = file_system.get_multi_fs(template_dirs) + if template_file.endswith("**"): + source_dir = template_file[:-3] + _, fs = multi_fs.which(source_dir) + if fs: + for a_triple in _listing_directory_files_recusively( + fs, source_dir, output + ): + yield a_triple else: reporter.report_error_message( "{0} cannot be found".format(template_file) ) - elif os.path.isdir(template_file_on_disk): - for a_triple in _list_dir_files( - template_file, template_file_on_disk, output - ): - yield a_triple else: - template_type = _get_template_type(template_file) - yield (template_file, output, template_type) - + _, fs = multi_fs.which(template_file) + if fs is None: + reporter.report_error_message( + "{0} cannot be found".format(template_file) + ) + elif fs.isdir(template_file): + for a_triple in _list_dir_files(fs, template_file, output): + yield a_triple + else: + template_type = _get_template_type(template_file) + yield (template_file, output, template_type) -def _list_dir_files(source, actual_source_path, dest): - for file_name in os.listdir(actual_source_path): - real_src_file = os.path.join(actual_source_path, file_name) - if os.path.isfile(real_src_file): - # please note jinja2 does NOT like windows path - # hence the following statement looks like cross platform - # src_file_under_dir = os.path.join(source, file_name) - # but actually it breaks windows instead. - src_file_under_dir = "%s/%s" % (source, file_name) - dest_file_under_dir = os.path.join(dest, file_name) +def _list_dir_files(fs, source, dest): + for file_name in fs.listdir(source): + # please note jinja2 does NOT like windows path + # hence the following statement looks like cross platform + # src_file_under_dir = os.path.join(source, file_name) + # but actually it breaks windows instead. + src_file_under_dir = "%s/%s" % (source, file_name) + if fs.isfile(src_file_under_dir): + dest_file_under_dir = dest + "/" + file_name template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) -def _listing_directory_files_recusively(source, actual_source_path, dest): - for file_name in os.listdir(actual_source_path): - src_file_under_dir = os.path.join(source, file_name) - dest_file_under_dir = os.path.join(dest, file_name) - real_src_file = os.path.join(actual_source_path, file_name) - if os.path.isfile(real_src_file): +def _listing_directory_files_recusively(fs, source, dest): + for file_name in fs.listdir(source): + src_file_under_dir = source + "/" + file_name + dest_file_under_dir = dest + "/" + file_name + if fs.isfile(src_file_under_dir): template_type = _get_template_type(src_file_under_dir) yield (src_file_under_dir, dest_file_under_dir, template_type) - elif os.path.isdir(real_src_file): + elif fs.isdir(src_file_under_dir): for a_triple in _listing_directory_files_recusively( - src_file_under_dir, real_src_file, dest_file_under_dir + fs, src_file_under_dir, dest_file_under_dir ): yield a_triple def _get_template_type(template_file): - _, extension = os.path.splitext(template_file) + _, extension = file_system.path_splitext(template_file) if extension: template_type = extension[1:] else: diff --git a/moban/plugins/__init__.py b/moban/plugins.py similarity index 81% rename from moban/plugins/__init__.py rename to moban/plugins.py index 5cfdf27e..88ce7f65 100644 --- a/moban/plugins/__init__.py +++ b/moban/plugins.py @@ -1,6 +1,5 @@ from moban import constants from lml.loader import scan_plugins_regex -from moban.plugins.template import MobanFactory BUILTIN_EXENSIONS = [ "moban.jinja2.engine", @@ -10,8 +9,5 @@ ] -ENGINES = MobanFactory() - - def make_sure_all_pkg_are_loaded(): scan_plugins_regex(constants.MOBAN_ALL, "moban", None, BUILTIN_EXENSIONS) diff --git a/moban/program_options.py b/moban/program_options.py new file mode 100644 index 00000000..efe0b0a7 --- /dev/null +++ b/moban/program_options.py @@ -0,0 +1,3 @@ +from moban import constants + +OPTIONS = {constants.CLI_DICT: {}} diff --git a/moban/utils.py b/moban/utils.py index 87e0f63b..98672fed 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,73 +1,10 @@ import os -import sys -import stat import errno import logging -from moban import constants, exceptions +from moban import constants, exceptions, file_system -log = logging.getLogger(__name__) -PY2 = sys.version_info[0] == 2 - - -def merge(left, right): - """ - deep merge dictionary on the left with the one - on the right. - - Fill in left dictionary with right one where - the value of the key from the right one in - the left one is missing or None. - """ - if isinstance(left, dict) and isinstance(right, dict): - for key, value in right.items(): - if key not in left: - left[key] = value - elif left[key] is None: - left[key] = value - else: - left[key] = merge(left[key], value) - return left - - -def search_file(base_dir, file_name): - the_file = file_name - if not os.path.exists(the_file): - if base_dir: - the_file = os.path.join(base_dir, file_name) - if not os.path.exists(the_file): - raise IOError( - constants.ERROR_DATA_FILE_NOT_FOUND % (file_name, the_file) - ) - else: - raise IOError(constants.ERROR_DATA_FILE_ABSENT % the_file) - return the_file - - -def file_permissions_copy(source, dest): - source_permissions = file_permissions(source) - dest_permissions = file_permissions(dest) - - if source_permissions != dest_permissions: - os.chmod(dest, source_permissions) - - -def file_permissions(afile): - if not os.path.exists(afile): - raise exceptions.FileNotFound(afile) - if sys.platform == "win32": - return 0 - return stat.S_IMODE(os.stat(afile).st_mode) - - -def write_file_out(filename, content): - if PY2 and content.__class__.__name__ == "unicode": - content = content.encode("utf-8") - dest_folder = os.path.dirname(filename) - if dest_folder: - mkdir_p(dest_folder) - with open(filename, "wb") as out: - out.write(content) +LOG = logging.getLogger(__name__) def mkdir_p(path): @@ -80,37 +17,17 @@ def mkdir_p(path): raise -def pip_install(packages): - import subprocess - - subprocess.check_call( - [sys.executable, "-m", "pip", "install", " ".join(packages)] - ) - - -def get_template_path(template_dirs, template): - temp_dir = "" - - for a_dir in template_dirs: - template_file_exists = os.path.exists( - os.path.join(a_dir, template) - ) and os.path.isfile(os.path.join(a_dir, template)) - - if template_file_exists: - temp_dir = a_dir - temp_file_path = os.path.join( - os.getcwd(), os.path.join(temp_dir, template) - ) - return temp_file_path - raise exceptions.FileNotFound - - def verify_the_existence_of_directories(dirs): + LOG.debug("Verifying the existence: %s", dirs) if not isinstance(dirs, list): dirs = [dirs] + results = [] + for directory in dirs: - if os.path.exists(directory): + + if file_system.exists(directory): + results.append(directory) continue should_I_ignore = ( constants.DEFAULT_CONFIGURATION_DIRNAME in directory @@ -121,15 +38,6 @@ def verify_the_existence_of_directories(dirs): pass else: raise exceptions.DirectoryNotFound( - constants.MESSAGE_DIR_NOT_EXIST % os.path.abspath(directory) + constants.MESSAGE_DIR_NOT_EXIST % directory ) - - -def find_file_in_template_dirs(src, template_dirs): - log.debug(template_dirs) - for folder in template_dirs: - path = os.path.join(folder, src) - if os.path.exists(path): - return path - else: - return None + return results diff --git a/requirements.txt b/requirements.txt index 2305c66a..8ad69281 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ ruamel.yaml>=0.15.5;python_version != '3.4' and python_version < '3.7' ruamel.yaml>=0.15.98;python_version == '3.8' jinja2>=2.7.1 lml>=0.0.9 -appdirs>=1.2.0 +appdirs>=1.4.3 crayons>= 0.1.0 -GitPython>=2.0.0 -git-url-parse>=1.2.2 +fs>=2.4.11 +jinja2-fsloader>=0.2.0 diff --git a/rnd_requirements.txt b/rnd_requirements.txt new file mode 100644 index 00000000..9e76ec64 --- /dev/null +++ b/rnd_requirements.txt @@ -0,0 +1,4 @@ +https://github.com/moremoban/moban-handlebars/archive/dev.zip + + + diff --git a/setup.py b/setup.py index a7564107..0c6abfaf 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ NAME = "moban" AUTHOR = "C. W." -VERSION = "0.5.0" +VERSION = "0.6.0" EMAIL = "wangc_2011@hotmail.com" LICENSE = "MIT" ENTRY_POINTS = { @@ -50,7 +50,7 @@ "Yet another jinja2 cli command for static text generation" ) URL = "https://github.com/moremoban/moban" -DOWNLOAD_URL = "%s/archive/0.5.0.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.0.tar.gz" % URL FILES = ["README.rst", "CONTRIBUTORS.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -78,10 +78,10 @@ INSTALL_REQUIRES = [ "jinja2>=2.7.1", "lml>=0.0.9", - "appdirs>=1.2.0", + "appdirs>=1.4.3", "crayons>= 0.1.0", - "GitPython>=2.0.0", - "git-url-parse>=1.2.2", + "fs>=2.4.11", + "jinja2-fsloader>=0.2.0", ] SETUP_COMMANDS = {} @@ -95,8 +95,8 @@ } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs moban v0.5.0 " + - "Find 0.5.0 in changelog for more details") +GS_COMMAND = ("gs moban v0.6.0 " + + "Find 0.6.0 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( diff --git a/tests/test_context.py b/tests/core/test_context.py similarity index 74% rename from tests/test_context.py rename to tests/core/test_context.py index 23b64133..846d2383 100644 --- a/tests/test_context.py +++ b/tests/core/test_context.py @@ -1,11 +1,12 @@ import os +import fs.path from nose.tools import eq_ -from moban.plugins.context import Context +from moban.core.context import Context def test_context(): - context = Context(os.path.join("tests", "fixtures")) + context = Context(fs.path.join("tests", "fixtures")) data = context.get_data("simple.yaml") eq_(data["simple"], "yaml") @@ -14,7 +15,7 @@ def test_environ_variables(): test_var = "TEST_ENVIRONMENT_VARIABLE" test_value = "am I found" os.environ[test_var] = test_value - context = Context(os.path.join("tests", "fixtures")) + context = Context(fs.path.join("tests", "fixtures")) data = context.get_data("simple.yaml") eq_(data[test_var], test_value) @@ -23,7 +24,7 @@ def test_json_data_overrides_environ_variables(): test_var = "TEST_ENVIRONMENT_VARIABLE" test_value = "am I found" os.environ[test_var] = test_value - context = Context(os.path.join("tests", "fixtures")) + context = Context(fs.path.join("tests", "fixtures")) data = context.get_data("simple.json") eq_(data[test_var], test_value) @@ -32,6 +33,6 @@ def test_unknown_data_file(): test_var = "TEST_ENVIRONMENT_VARIABLE" test_value = "am I found" os.environ[test_var] = test_value - context = Context(os.path.join("tests", "fixtures")) + context = Context(fs.path.join("tests", "fixtures")) data = context.get_data("unknown.data") eq_(data[test_var], test_value) diff --git a/tests/test_template.py b/tests/core/test_engine.py similarity index 86% rename from tests/test_template.py rename to tests/core/test_engine.py index 4c495195..d1c6c790 100644 --- a/tests/test_template.py +++ b/tests/core/test_engine.py @@ -1,13 +1,14 @@ import os +import fs.path from mock import patch +from moban.core import ENGINES from nose.tools import eq_ -from moban.plugins import ENGINES from moban.definitions import TemplateTarget from moban.jinja2.engine import Engine from moban.data_loaders.yaml import open_yaml -MODULE = "moban.plugins.template" +MODULE = "moban.core.moban_factory" @patch(MODULE + ".MobanEngine._render_with_finding_data_first") @@ -57,12 +58,12 @@ def test_do_templates_2(_do_templates_with_more_shared_templates): def test_do_templates_with_more_shared_templates(): - base_dir = os.path.join("tests", "fixtures") + base_dir = fs.path.join("tests", "fixtures") engine = ENGINES.get_engine( - "jinja2", base_dir, os.path.join(base_dir, "config") + "jinja2", base_dir, fs.path.join(base_dir, "config") ) engine._render_with_finding_template_first( - {"a.jj2": [(os.path.join(base_dir, "child.yaml"), "test")]} + {"a.jj2": [(fs.path.join(base_dir, "child.yaml"), "test")]} ) with open("test", "r") as f: content = f.read() @@ -71,12 +72,12 @@ def test_do_templates_with_more_shared_templates(): def test_do_templates_with_more_shared_data(): - base_dir = os.path.join("tests", "fixtures") + base_dir = fs.path.join("tests", "fixtures") engine = ENGINES.get_engine( - "jinja2", base_dir, os.path.join(base_dir, "config") + "jinja2", base_dir, fs.path.join(base_dir, "config") ) engine._render_with_finding_data_first( - {os.path.join(base_dir, "child.yaml"): [("a.jj2", "test")]} + {fs.path.join(base_dir, "child.yaml"): [("a.jj2", "test")]} ) with open("test", "r") as f: content = f.read() @@ -85,7 +86,7 @@ def test_do_templates_with_more_shared_data(): def test_get_user_defined_engine(): - test_fixture = os.path.join( + test_fixture = fs.path.join( "tests", "fixtures", "mobanengine", "sample_template_type.yml" ) template_types = open_yaml(test_fixture) @@ -95,7 +96,7 @@ def test_get_user_defined_engine(): def test_custom_file_extension_is_assocated_with_user_defined_engine(): - test_fixture = os.path.join( + test_fixture = fs.path.join( "tests", "fixtures", "mobanengine", "sample_template_type.yml" ) template_types = open_yaml(test_fixture) @@ -105,7 +106,7 @@ def test_custom_file_extension_is_assocated_with_user_defined_engine(): def test_built_in_jinja2_file_extension_still_works(): - test_fixture = os.path.join( + test_fixture = fs.path.join( "tests", "fixtures", "mobanengine", "sample_template_type.yml" ) template_types = open_yaml(test_fixture) diff --git a/tests/test_engine.py b/tests/core/test_moban_factory.py similarity index 82% rename from tests/test_engine.py rename to tests/core/test_moban_factory.py index 803258fa..c5f4b179 100644 --- a/tests/test_engine.py +++ b/tests/core/test_moban_factory.py @@ -1,26 +1,27 @@ import os import sys +import fs.path import moban.exceptions as exceptions from mock import patch from lml.plugin import PluginInfo +from moban.core import ENGINES from nose.tools import eq_, raises -from moban.plugins import ENGINES +from moban.core.context import Context from moban.jinja2.engine import ( Engine, is_extension_list_valid, import_module_of_extension, ) -from moban.plugins.context import Context -from moban.plugins.template import MobanEngine, expand_template_directories +from moban.core.moban_factory import MobanEngine, expand_template_directories -USER_HOME = os.path.join("user", "home", ".moban", "repos") +USER_HOME = fs.path.join("user", "home", ".moban", "repos") @PluginInfo("library", tags=["testmobans"]) class TestPypkg: def __init__(self): - __package_path__ = os.path.dirname(__file__) + __package_path__ = os.path.normcase(os.path.dirname(__file__)) self.resources_path = os.path.join(__package_path__, "fixtures") @@ -30,12 +31,12 @@ def test_expand_pypi_dir(): assert os.path.exists(directory) -@patch("moban.repo.get_moban_home", return_value=USER_HOME) -@patch("os.path.exists", return_value=True) +@patch("moban.deprecated.repo.get_moban_home", return_value=USER_HOME) +@patch("moban.file_system.exists", return_value=True) def test_expand_repo_dir(_, __): dirs = list(expand_template_directories("git_repo:template")) - expected = [os.path.join(USER_HOME, "git_repo", "template")] + expected = [fs.path.join(USER_HOME, "git_repo", "template")] eq_(expected, dirs) @@ -62,7 +63,7 @@ def test_unknown_template_type(): @raises(exceptions.DirectoryNotFound) def test_non_existent_tmpl_directries(): - MobanEngine("abc", "tests", Engine) + ENGINES.get_engine("jj2", "idontexist", "") @raises(exceptions.DirectoryNotFound) @@ -77,7 +78,7 @@ def test_non_existent_ctx_directries(): def test_file_tests(): output = "test.txt" - path = os.path.join("tests", "fixtures", "jinja_tests") + path = fs.path.join("tests", "fixtures", "jinja_tests") engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("file_tests.template", "file_tests.yml", output) with open(output, "r") as output_file: @@ -88,7 +89,7 @@ def test_file_tests(): def test_global_template_variables(): output = "test.txt" - path = os.path.join("tests", "fixtures", "globals") + path = fs.path.join("tests", "fixtures", "globals") engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("variables.template", "variables.yml", output) with open(output, "r") as output_file: @@ -99,7 +100,7 @@ def test_global_template_variables(): def test_nested_global_template_variables(): output = "test.txt" - path = os.path.join("tests", "fixtures", "globals") + path = fs.path.join("tests", "fixtures", "globals") engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("nested.template", "variables.yml", output) with open(output, "r") as output_file: @@ -113,7 +114,7 @@ def test_environ_variables_as_data(): test_value = "foo" os.environ[test_var] = test_value output = "test.txt" - path = os.path.join("tests", "fixtures", "environ_vars_as_data") + path = fs.path.join("tests", "fixtures", "environ_vars_as_data") engine = ENGINES.get_engine("jinja2", [path], path) engine.render_to_file("test.template", "this_does_not_exist.yml", output) with open(output, "r") as output_file: @@ -124,7 +125,7 @@ def test_environ_variables_as_data(): def test_string_template(): output = "test.txt" - path = os.path.join("tests", "fixtures") + path = fs.path.join("tests", "fixtures") engine = ENGINES.get_engine("jinja2", [path], path) engine.render_string_to_file("{{simple}}", "simple.yaml", output) with open(output, "r") as output_file: diff --git a/tests/test_json_loader.py b/tests/data_loaders/test_json_loader.py similarity index 70% rename from tests/test_json_loader.py rename to tests/data_loaders/test_json_loader.py index 8a05ef9a..44ed74fa 100644 --- a/tests/test_json_loader.py +++ b/tests/data_loaders/test_json_loader.py @@ -1,10 +1,9 @@ -import os - +import fs.path from nose.tools import eq_ from moban.data_loaders.json_loader import open_json def test_open_json(): - content = open_json(os.path.join("tests", "fixtures", "child.json")) + content = open_json(fs.path.join("tests", "fixtures", "child.json")) expected = {"key": "hello world", "pass": "ox"} eq_(expected, content) diff --git a/tests/test_merge_dict.py b/tests/data_loaders/test_merge_dict.py similarity index 95% rename from tests/test_merge_dict.py rename to tests/data_loaders/test_merge_dict.py index 55cc01d2..445f62db 100644 --- a/tests/test_merge_dict.py +++ b/tests/data_loaders/test_merge_dict.py @@ -1,4 +1,4 @@ -from moban.utils import merge +from moban.data_loaders.manager import merge def test_simple_union(): diff --git a/tests/data_loaders/test_overrides.py b/tests/data_loaders/test_overrides.py new file mode 100644 index 00000000..d9b190d3 --- /dev/null +++ b/tests/data_loaders/test_overrides.py @@ -0,0 +1,80 @@ +import os + +from nose.tools import eq_ +from moban.data_loaders.manager import load_data + + +def test_overrides_a_list_of_config_files(): + base_dir = os.path.join("tests", "fixtures", "issue_126") + config_dir = os.path.join(base_dir, "config") + actual = load_data(config_dir, os.path.join(base_dir, "the_config.yaml")) + expected = [ + ("key", "value"), + ("key_from_a", "apple"), + ("key_from_b", "bee"), + ] + for item, expected_item in zip(actual.items(), expected): + eq_(item, expected_item) + + +def test_overrides_ignores_override_sequence(): + base_dir = os.path.join("tests", "fixtures", "issue_126") + config_dir = os.path.join(base_dir, "config") + actual = load_data(config_dir, os.path.join(base_dir, "the_config.yaml")) + expected = [ + ("key", "value"), + ("key_from_a", "apple"), + ("key_from_b", "bee"), + ] + for item, expected_item in zip(actual.items(), expected): + eq_(item, expected_item) + + +def test_overrides_select_keys_from_parent_files(): + base_dir = os.path.join("tests", "fixtures", "issue_126") + config_dir = os.path.join(base_dir, "config") + actual = load_data( + config_dir, os.path.join(base_dir, "multi-key-config.yaml") + ) + expected = [ + ("cat", "from config"), + ("alpha", "from a"), + ("beta", "from b"), + ] + for item, expected_item in zip(actual.items(), expected): + eq_(item, expected_item) + + +def test_overrides_select_keys(): + base_dir = os.path.join("tests", "fixtures", "issue_126") + config_dir = os.path.join(base_dir, "config") + actual = load_data( + config_dir, os.path.join(base_dir, "multi-key-config-override.yaml") + ) + expected = [ + ("alpha", "from config"), + ("cat", "from config"), + ("beta", "from b"), + ] + for item, expected_item in zip(actual.items(), expected): + eq_(item, expected_item) + + +def test_overrides_nested_keys(): + base_dir = os.path.join("tests", "fixtures", "issue_126") + config_dir = os.path.join(base_dir, "config") + actual = load_data(config_dir, os.path.join(base_dir, "raspberry.yaml")) + expected = { + "raspberry": { + "other": "OpenGL 3.0", + "version": 4, + "memory": "4GB", + "core": "quad", + "wifi": "2.5 & 5.0 GHz", + "USB": 3.0, + "Bluetooth": 5.0, + }, + "tessel": {"version": 2, "USB": "micro", "wifi": "802.11gn"}, + } + + eq_(dict(actual), expected) diff --git a/tests/data_loaders/test_yaml_loader.py b/tests/data_loaders/test_yaml_loader.py new file mode 100644 index 00000000..c8f2c8b1 --- /dev/null +++ b/tests/data_loaders/test_yaml_loader.py @@ -0,0 +1,34 @@ +import fs.path +from nose.tools import eq_, raises +from moban.data_loaders.yaml import open_yaml +from moban.data_loaders.manager import load_data + + +def test_simple_yaml(): + test_file = fs.path.join("tests", "fixtures", "simple.yaml") + data = open_yaml(test_file) + eq_(data, {"simple": "yaml"}) + + +def test_inheritance_yaml(): + test_file = fs.path.join("tests", "fixtures", "child.yaml") + data = load_data(fs.path.join("tests", "fixtures", "config"), test_file) + eq_(data, {"key": "hello world", "pass": "ox"}) + + +@raises(IOError) +def test_exception(): + test_file = fs.path.join("tests", "fixtures", "orphan.yaml") + load_data(fs.path.join("tests", "fixtures", "config"), test_file) + + +@raises(IOError) +def test_exception_2(): + test_file = fs.path.join("tests", "fixtures", "dragon.yaml") + load_data(fs.path.join("tests", "fixtures", "config"), test_file) + + +@raises(IOError) +def test_exception_3(): + test_file = fs.path.join("tests", "fixtures", "dragon.yaml") + load_data(None, test_file) diff --git a/tests/deprecated/test_handle_requires.py b/tests/deprecated/test_handle_requires.py new file mode 100644 index 00000000..fa427c8f --- /dev/null +++ b/tests/deprecated/test_handle_requires.py @@ -0,0 +1,77 @@ +from mock import patch +from nose.tools import eq_ +from moban.deprecated import GitRequire + + +@patch("moban.deprecated.pip_install") +def test_handle_requires_pypkg(fake_pip_install): + modules = ["package1", "package2"] + from moban.deprecated import handle_requires + + handle_requires(modules) + fake_pip_install.assert_called_with(modules) + + +@patch("moban.deprecated.pip_install") +def test_handle_requires_pypkg_with_alternative_syntax(fake_pip_install): + modules = [{"type": "pypi", "name": "pypi-mobans"}] + from moban.mobanfile import handle_requires + + handle_requires(modules) + fake_pip_install.assert_called_with(["pypi-mobans"]) + + +@patch("moban.deprecated.git_clone") +def test_handle_requires_repos(fake_git_clone): + repos = ["https://github.com/my/repo", "https://gitlab.com/my/repo"] + from moban.mobanfile import handle_requires + + expected = [] + for repo in repos: + expected.append(GitRequire(git_url=repo, submodule=False)) + + handle_requires(repos) + fake_git_clone.assert_called_with(expected) + + +@patch("moban.deprecated.git_clone") +def test_handle_requires_repos_with_alternative_syntax(fake_git_clone): + repos = [{"type": "git", "url": "https://github.com/my/repo"}] + from moban.mobanfile import handle_requires + + handle_requires(repos) + fake_git_clone.assert_called_with( + [GitRequire(git_url="https://github.com/my/repo")] + ) + + +@patch("moban.deprecated.pip_install") +@patch("moban.deprecated.git_clone") +def test_handle_requires_repos_with_submodule( + fake_git_clone, fake_pip_install +): + repos = [ + {"type": "git", "url": "https://github.com/my/repo", "submodule": True} + ] + from moban.mobanfile import handle_requires + + handle_requires(repos) + fake_git_clone.assert_called_with( + [GitRequire(git_url="https://github.com/my/repo", submodule=True)] + ) + eq_(fake_pip_install.called, False) + + +def test_is_repo(): + repos = [ + "https://github.com/my/repo", + "https://gitlab.com/my/repo", + "https://bitbucket.com/my/repo", + "https://unsupported.com/my/repo", + "invalid/repo/url", + ] + from moban.deprecated import is_repo + + actual = [is_repo(repo) for repo in repos] + expected = [True, True, True, False, False] + eq_(expected, actual) diff --git a/tests/test_repo.py b/tests/deprecated/test_repo.py similarity index 95% rename from tests/test_repo.py rename to tests/deprecated/test_repo.py index 1c82e258..1cfc571b 100644 --- a/tests/test_repo.py +++ b/tests/deprecated/test_repo.py @@ -1,20 +1,19 @@ -import os - +import fs.path from mock import patch -from moban.repo import ( +from nose.tools import eq_, raises +from moban.deprecated import GitRequire +from moban.exceptions import NoGitCommand +from moban.deprecated.repo import ( git_clone, get_repo_name, get_moban_home, make_sure_git_is_available, ) -from nose.tools import eq_, raises -from moban.exceptions import NoGitCommand -from moban.definitions import GitRequire @patch("appdirs.user_cache_dir", return_value="root") @patch("moban.utils.mkdir_p") -@patch("os.path.exists") +@patch("moban.file_system.exists") @patch("git.Repo", autospec=True) class TestGitFunctions: def setUp(self): @@ -30,7 +29,7 @@ def setUp(self): self.require_with_reference = GitRequire( git_url=self.repo, reference="a-commit-reference" ) - self.expected_local_repo_path = os.path.join( + self.expected_local_repo_path = fs.path.join( "root", "repos", self.repo_name ) @@ -152,7 +151,7 @@ def test_get_repo_name_can_handle_invalid_url(fake_reporter): @patch("appdirs.user_cache_dir", return_value="root") def test_get_moban_home(_): actual = get_moban_home() - eq_(os.path.join("root", "repos"), actual) + eq_(fs.path.join("root", "repos"), actual) @raises(NoGitCommand) diff --git a/tests/fixtures/.moban-version-1.0.yml b/tests/fixtures/.moban-version-1.0.yml index 0e51a764..04ca9eb9 100644 --- a/tests/fixtures/.moban-version-1.0.yml +++ b/tests/fixtures/.moban-version-1.0.yml @@ -2,5 +2,4 @@ version: 1.0 configuration: configuration_dir: "commons/config" template_dir: - - "commons/templates" - ".moban.d" diff --git a/tests/fixtures/.moban-version-1234.yml b/tests/fixtures/.moban-version-1234.yml index 4c265181..73d0d55d 100644 --- a/tests/fixtures/.moban-version-1234.yml +++ b/tests/fixtures/.moban-version-1234.yml @@ -1,8 +1,7 @@ moban_file_spec_version: 1234 configuration: - configuration_dir: "commons/config" + configuration_dir: "." template_dir: - - "commons/templates" - ".moban.d" configuration: data.yaml targets: diff --git a/tests/fixtures/child.yaml b/tests/fixtures/child.yaml index 4b082225..ed31c83b 100644 --- a/tests/fixtures/child.yaml +++ b/tests/fixtures/child.yaml @@ -1,2 +1,2 @@ +key: 'hello world' overrides: base.yaml -key: hello world diff --git a/tests/fixtures/config/base.yaml b/tests/fixtures/config/base.yaml index 0270c58b..1ef517a0 100644 --- a/tests/fixtures/config/base.yaml +++ b/tests/fixtures/config/base.yaml @@ -1,2 +1,2 @@ -key: to be overriden +key: 'to be overriden' pass: ox diff --git a/tests/fixtures/duplicated.moban.yml b/tests/fixtures/duplicated.moban.yml index 2d4407b1..e2b4c0c6 100644 --- a/tests/fixtures/duplicated.moban.yml +++ b/tests/fixtures/duplicated.moban.yml @@ -1,8 +1,6 @@ configuration: - configuration_dir: "commons/config" template_dir: - - "commons/templates" - - ".moban.d" + - "." targets: - setup.py: setup.py - setup.py: setup.py diff --git a/tests/fixtures/file_system/template-sources.tar b/tests/fixtures/file_system/template-sources.tar new file mode 100644 index 00000000..2f13227f Binary files /dev/null and b/tests/fixtures/file_system/template-sources.tar differ diff --git a/tests/fixtures/file_system/template-sources.zip b/tests/fixtures/file_system/template-sources.zip new file mode 100644 index 00000000..5085110c Binary files /dev/null and b/tests/fixtures/file_system/template-sources.zip differ diff --git a/tests/fixtures/issue_126/config/A.yaml b/tests/fixtures/issue_126/config/A.yaml new file mode 100644 index 00000000..3ec33513 --- /dev/null +++ b/tests/fixtures/issue_126/config/A.yaml @@ -0,0 +1 @@ +key_from_a: apple diff --git a/tests/fixtures/issue_126/config/B.yaml b/tests/fixtures/issue_126/config/B.yaml new file mode 100644 index 00000000..9b200839 --- /dev/null +++ b/tests/fixtures/issue_126/config/B.yaml @@ -0,0 +1 @@ +key_from_b: bee diff --git a/tests/fixtures/issue_126/config/multi-key-A.yaml b/tests/fixtures/issue_126/config/multi-key-A.yaml new file mode 100644 index 00000000..79765e75 --- /dev/null +++ b/tests/fixtures/issue_126/config/multi-key-A.yaml @@ -0,0 +1,2 @@ +alpha: 'from a' +beta: 'from a' diff --git a/tests/fixtures/issue_126/config/multi-key-B.yaml b/tests/fixtures/issue_126/config/multi-key-B.yaml new file mode 100644 index 00000000..4ede311c --- /dev/null +++ b/tests/fixtures/issue_126/config/multi-key-B.yaml @@ -0,0 +1,2 @@ +alpha: 'from b' +beta: 'from b' diff --git a/tests/fixtures/issue_126/config/multi-key-config.yaml b/tests/fixtures/issue_126/config/multi-key-config.yaml new file mode 100644 index 00000000..43973344 --- /dev/null +++ b/tests/fixtures/issue_126/config/multi-key-config.yaml @@ -0,0 +1,4 @@ +cat: 'from config' +overrides: + - multi-key-A.yaml:alpha + - multi-key-B.yaml:beta diff --git a/tests/fixtures/issue_126/config/nested-A.yaml b/tests/fixtures/issue_126/config/nested-A.yaml new file mode 100644 index 00000000..9529fb1e --- /dev/null +++ b/tests/fixtures/issue_126/config/nested-A.yaml @@ -0,0 +1,6 @@ +raspberry: + core: quad + memory: 4GB + version: 4 +tessel: + version: 2 diff --git a/tests/fixtures/issue_126/config/nested-B.yaml b/tests/fixtures/issue_126/config/nested-B.yaml new file mode 100644 index 00000000..5c48e1a3 --- /dev/null +++ b/tests/fixtures/issue_126/config/nested-B.yaml @@ -0,0 +1,7 @@ +raspberry: + Bluetooth: 5.0 + USB: 3.0 + wifi: '2.5 & 5.0 GHz' +tessel: + USB: micro + wifi: 802.11gn diff --git a/tests/fixtures/issue_126/config_BA.yaml b/tests/fixtures/issue_126/config_BA.yaml new file mode 100644 index 00000000..823afb2b --- /dev/null +++ b/tests/fixtures/issue_126/config_BA.yaml @@ -0,0 +1,4 @@ +key: value +overrides: + - B.yaml + - A.yaml diff --git a/tests/fixtures/issue_126/multi-key-config-override.yaml b/tests/fixtures/issue_126/multi-key-config-override.yaml new file mode 100644 index 00000000..0f9c8024 --- /dev/null +++ b/tests/fixtures/issue_126/multi-key-config-override.yaml @@ -0,0 +1,5 @@ +alpha: 'from config' +cat: 'from config' +overrides: + - multi-key-A.yaml:alpha + - multi-key-B.yaml:beta diff --git a/tests/fixtures/issue_126/multi-key-config.yaml b/tests/fixtures/issue_126/multi-key-config.yaml new file mode 100644 index 00000000..43973344 --- /dev/null +++ b/tests/fixtures/issue_126/multi-key-config.yaml @@ -0,0 +1,4 @@ +cat: 'from config' +overrides: + - multi-key-A.yaml:alpha + - multi-key-B.yaml:beta diff --git a/tests/fixtures/issue_126/raspberry.yaml b/tests/fixtures/issue_126/raspberry.yaml new file mode 100644 index 00000000..feadf485 --- /dev/null +++ b/tests/fixtures/issue_126/raspberry.yaml @@ -0,0 +1,5 @@ +overrides: + - nested-A.yaml + - nested-B.yaml +raspberry: + other: 'OpenGL 3.0' diff --git a/tests/fixtures/issue_126/the_config.yaml b/tests/fixtures/issue_126/the_config.yaml new file mode 100644 index 00000000..d56563ff --- /dev/null +++ b/tests/fixtures/issue_126/the_config.yaml @@ -0,0 +1,4 @@ +key: value +overrides: + - A.yaml + - B.yaml diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index 2c141bf6..302b99ff 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -17,16 +17,17 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_file") - def test_custom_options(self, fake_template_doer): + @patch("moban.file_system.abspath") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") + def test_custom_options(self, fake_template_doer, fake_abspath): test_args = [ "moban", "-c", self.config_file, "-cd", - "/home/developer/configuration", + ".", "-td", - "/home/developer/templates", + ".", "-t", "a.jj2", ] @@ -38,7 +39,7 @@ def test_custom_options(self, fake_template_doer): "a.jj2", "config.yaml", "moban.output" ) - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_minimal_options(self, fake_template_doer): test_args = ["moban", "-c", self.config_file, "-t", "a.jj2"] with patch.object(sys, "argv", test_args): @@ -72,7 +73,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_default_options(self, fake_template_doer): test_args = ["moban", "-t", "a.jj2"] with patch.object(sys, "argv", test_args): @@ -83,7 +84,7 @@ def test_default_options(self, fake_template_doer): "a.jj2", "data.yml", "moban.output" ) - @patch("moban.plugins.template.MobanEngine.render_string_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_string_to_file") def test_string_template(self, fake_template_doer): string_template = "{{HELLO}}" test_args = ["moban", string_template] @@ -132,7 +133,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command(self, fake_template_doer): test_args = ["moban"] with patch.object(sys, "argv", test_args): @@ -151,7 +152,7 @@ def test_single_command(self, fake_template_doer): ) @raises(Exception) - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command_with_missing_output(self, fake_template_doer): test_args = ["moban", "-t", "README.rst.jj2"] with patch.object(sys, "argv", test_args): @@ -159,20 +160,21 @@ def test_single_command_with_missing_output(self, fake_template_doer): main() - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command_with_a_few_options(self, fake_template_doer): test_args = ["moban", "-t", "README.rst.jj2", "-o", "xyz.output"] with patch.object(sys, "argv", test_args): from moban.main import main main() + call_args = list(fake_template_doer.call_args[0][0]) eq_( call_args, [TemplateTarget("README.rst.jj2", "data.yaml", "xyz.output")], ) - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command_with_options(self, fake_template_doer): test_args = [ "moban", @@ -222,7 +224,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command(self, fake_template_doer): test_args = ["moban"] with patch.object(sys, "argv", test_args): @@ -260,7 +262,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_single_command(self, fake_template_doer): test_args = ["moban", "-m", self.config_file] with patch.object(sys, "argv", test_args): @@ -295,7 +297,7 @@ def setUp(self): ) self.patcher1.start() - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_template_option_override_moban_file(self, fake_template_doer): test_args = ["moban", "-t", "setup.py.jj2"] with patch.object(sys, "argv", test_args): @@ -306,7 +308,7 @@ def test_template_option_override_moban_file(self, fake_template_doer): "setup.py.jj2", "data.yml", "moban.output" ) - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_template_option_not_in_moban_file(self, fake_template_doer): test_args = ["moban", "-t", "foo.jj2"] with patch.object(sys, "argv", test_args): @@ -339,7 +341,7 @@ def setUp(self): self.config_file = ".moban.yml" @raises(SystemExit) - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_no_configuration(self, fake_template_doer): with open(self.config_file, "w") as f: f.write("") @@ -350,7 +352,7 @@ def test_no_configuration(self, fake_template_doer): main() @raises(SystemExit) - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_no_configuration_2(self, fake_template_doer): with open(self.config_file, "w") as f: f.write("not: related") @@ -361,7 +363,7 @@ def test_no_configuration_2(self, fake_template_doer): main() @raises(SystemExit) - @patch("moban.plugins.template.MobanEngine.render_to_files") + @patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_no_targets(self, fake_template_doer): with open(self.config_file, "w") as f: f.write("configuration: test") @@ -385,16 +387,14 @@ def setUp(self): with open(self.data_file, "w") as f: f.write("hello: world") - @patch( - "moban.utils.verify_the_existence_of_directories", return_value=True - ) + @patch("moban.utils.verify_the_existence_of_directories", return_value=".") def test_single_command(self, _): test_args = ["moban"] with patch.object(sys, "argv", test_args): from moban.main import main with patch( - "moban.plugins.template.MobanEngine.render_to_files" + "moban.core.moban_factory.MobanEngine.render_to_files" ) as fake: main() call_args = list(fake.call_args[0][0]) @@ -419,7 +419,7 @@ def setUp(self): with open(self.config_file, "w") as f: f.write("hello: world") - @patch("moban.plugins.template.MobanEngine.render_to_file") + @patch("moban.core.moban_factory.MobanEngine.render_to_file") def test_mako_option(self, fake_template_doer): test_args = ["moban", "-t", "a.mako"] with patch.object(sys, "argv", test_args): @@ -436,8 +436,66 @@ def tearDown(self): @raises(SystemExit) def test_version_option(): - test_args = ["moban", "-v"] + test_args = ["moban", "-V"] + with patch.object(sys, "argv", test_args): + from moban.main import main + + main() + + +@patch("logging.basicConfig") +def test_debug_option(fake_config): + fake_config.side_effect = [IOError("stop test")] + test_args = ["moban", "-vv"] + with patch.object(sys, "argv", test_args): + from moban.main import main + + try: + main() + except IOError: + fake_config.assert_called_with( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + level=10, + ) + + +@patch("moban.utils.verify_the_existence_of_directories", return_value=[]) +def test_git_repo_example(_): + test_args = [ + "moban", + "-t", + "git://github.com/moremoban/pypi-mobans.git!/templates/_version.py.jj2", + "-c", + "git://github.com/moremoban/pypi-mobans.git!/config/data.yml", + "-o", + "test_git_repo_example.py", + ] + with patch.object(sys, "argv", test_args): + from moban.main import main + + main() + with open("test_git_repo_example.py") as f: + content = f.read() + eq_(content, '__version__ = "0.1.1rc3"\n__author__ = "C.W."') + os.unlink("test_git_repo_example.py") + + +@patch("moban.utils.verify_the_existence_of_directories", return_value=[]) +def test_pypi_pkg_example(_): + test_args = [ + "moban", + "-t", + "pypi://pypi-mobans-pkg/resources/templates/_version.py.jj2", + "-c", + "pypi://pypi-mobans-pkg/resources/config/data.yml", + "-o", + "test_pypi_pkg_example.py", + ] with patch.object(sys, "argv", test_args): from moban.main import main main() + with open("test_pypi_pkg_example.py") as f: + content = f.read() + eq_(content, '__version__ = "0.1.1rc3"\n__author__ = "C.W."') + os.unlink("test_pypi_pkg_example.py") diff --git a/tests/test_jinja2_engine.py b/tests/jinja2/test_engine.py similarity index 80% rename from tests/test_jinja2_engine.py rename to tests/jinja2/test_engine.py index 8996d951..5238621a 100644 --- a/tests/test_jinja2_engine.py +++ b/tests/jinja2/test_engine.py @@ -1,12 +1,14 @@ import os +from moban import file_system from nose.tools import eq_ from moban.jinja2.engine import Engine def test_jinja2_template(): path = os.path.join("tests", "fixtures", "jinja_tests") - engine = Engine([path]) + fs = file_system.get_multi_fs([path]) + engine = Engine(fs) template = engine.get_template("file_tests.template") data = dict(test="here") result = engine.apply_template(template, data, None) @@ -16,7 +18,8 @@ def test_jinja2_template(): def test_jinja2_template_string(): path = os.path.join("tests", "fixtures", "jinja_tests") - engine = Engine([path]) + fs = file_system.get_multi_fs([path]) + engine = Engine(fs) template = engine.get_template_from_string("{{test}}") data = dict(test="here") result = engine.apply_template(template, data, None) diff --git a/tests/test_jinja2_extensions.py b/tests/jinja2/test_extensions.py similarity index 72% rename from tests/test_jinja2_extensions.py rename to tests/jinja2/test_extensions.py index 5ef492ba..61ddbf1a 100644 --- a/tests/test_jinja2_extensions.py +++ b/tests/jinja2/test_extensions.py @@ -1,9 +1,10 @@ import os +from moban import file_system from nose.tools import eq_ from moban.jinja2.engine import Engine -from moban.plugins.template import MobanEngine from moban.jinja2.extensions import jinja_global +from moban.core.moban_factory import MobanEngine def test_globals(): @@ -11,7 +12,8 @@ def test_globals(): test_dict = dict(hello="world") jinja_global("test", test_dict) path = os.path.join("tests", "fixtures", "globals") - engine = MobanEngine([path], path, Engine([path])) + template_fs = file_system.get_multi_fs([path]) + engine = MobanEngine(template_fs, path, Engine(template_fs)) engine.render_to_file("basic.template", "basic.yml", output) with open(output, "r") as output_file: content = output_file.read() diff --git a/tests/test_filter_github.py b/tests/jinja2/test_github.py similarity index 100% rename from tests/test_filter_github.py rename to tests/jinja2/test_github.py diff --git a/tests/test_filter_repr.py b/tests/jinja2/test_repr.py similarity index 100% rename from tests/test_filter_repr.py rename to tests/jinja2/test_repr.py diff --git a/tests/test_text_filter.py b/tests/jinja2/test_text.py similarity index 100% rename from tests/test_text_filter.py rename to tests/jinja2/test_text.py diff --git a/tests/mobanfile/test_mobanfile.py b/tests/mobanfile/test_mobanfile.py index 4592c76c..db8c11b7 100644 --- a/tests/mobanfile/test_mobanfile.py +++ b/tests/mobanfile/test_mobanfile.py @@ -1,131 +1,24 @@ -import os - +import fs.path from mock import patch from nose.tools import eq_ -from moban.definitions import GitRequire, TemplateTarget - - -class TestFinder: - def setUp(self): - self.patcher = patch("os.path.exists") - self.fake_file_existence = self.patcher.start() - self.fake_file_existence.__name__ = "fake" - self.fake_file_existence.__module__ = "fake" - - def tearDown(self): - self.patcher.stop() - - def test_moban_yml(self): - self.fake_file_existence.return_value = True - from moban.mobanfile import find_default_moban_file - - actual = find_default_moban_file() - eq_(".moban.yml", actual) - - def test_moban_yaml(self): - self.fake_file_existence.side_effect = [False, True] - from moban.mobanfile import find_default_moban_file - - actual = find_default_moban_file() - eq_(".moban.yaml", actual) - - def test_no_moban_file(self): - self.fake_file_existence.side_effect = [False, False] - from moban.mobanfile import find_default_moban_file - - actual = find_default_moban_file() - assert actual is None - - -@patch("moban.mobanfile.pip_install") -def test_handle_requires_pypkg(fake_pip_install): - modules = ["package1", "package2"] - from moban.mobanfile import handle_requires - - handle_requires(modules) - fake_pip_install.assert_called_with(modules) - - -@patch("moban.mobanfile.pip_install") -def test_handle_requires_pypkg_with_alternative_syntax(fake_pip_install): - modules = [{"type": "pypi", "name": "pypi-mobans"}] - from moban.mobanfile import handle_requires - - handle_requires(modules) - fake_pip_install.assert_called_with(["pypi-mobans"]) - - -@patch("moban.mobanfile.git_clone") -def test_handle_requires_repos(fake_git_clone): - repos = ["https://github.com/my/repo", "https://gitlab.com/my/repo"] - from moban.mobanfile import handle_requires - - expected = [] - for repo in repos: - expected.append(GitRequire(git_url=repo, submodule=False)) - - handle_requires(repos) - fake_git_clone.assert_called_with(expected) - - -@patch("moban.mobanfile.git_clone") -def test_handle_requires_repos_with_alternative_syntax(fake_git_clone): - repos = [{"type": "git", "url": "https://github.com/my/repo"}] - from moban.mobanfile import handle_requires - - handle_requires(repos) - fake_git_clone.assert_called_with( - [GitRequire(git_url="https://github.com/my/repo")] - ) - - -@patch("moban.mobanfile.pip_install") -@patch("moban.mobanfile.git_clone") -def test_handle_requires_repos_with_submodule( - fake_git_clone, fake_pip_install -): - repos = [ - {"type": "git", "url": "https://github.com/my/repo", "submodule": True} - ] - from moban.mobanfile import handle_requires - - handle_requires(repos) - fake_git_clone.assert_called_with( - [GitRequire(git_url="https://github.com/my/repo", submodule=True)] - ) - eq_(fake_pip_install.called, False) - - -def test_is_repo(): - repos = [ - "https://github.com/my/repo", - "https://gitlab.com/my/repo", - "https://bitbucket.com/my/repo", - "https://unsupported.com/my/repo", - "invalid/repo/url", - ] - from moban.mobanfile import is_repo - - actual = [is_repo(repo) for repo in repos] - expected = [True, True, True, False, False] - eq_(expected, actual) +from moban.definitions import TemplateTarget -@patch("moban.plugins.template.MobanEngine.render_to_files") +@patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_handle_targets(fake_renderer): from moban.mobanfile import handle_targets TEMPLATE = "copier-test01.csv" OUTPUT = "output.csv" CONFIGURATION = "child.yaml" - TEMPLATE_DIRS = [os.path.join("tests", "fixtures")] + TEMPLATE_DIRS = [fs.path.join("tests", "fixtures")] DEFAULT_TEMPLATE_TYPE = "jinja2" options = dict( configuration=CONFIGURATION, template_type=DEFAULT_TEMPLATE_TYPE, template_dir=TEMPLATE_DIRS, - configuration_dir=os.path.join("tests", "fixtures"), + configuration_dir=fs.path.join("tests", "fixtures"), ) short_hand_targets = [{OUTPUT: TEMPLATE}] handle_targets(options, short_hand_targets) @@ -144,7 +37,7 @@ def test_handle_targets(fake_renderer): ) -@patch("moban.plugins.template.MobanEngine.render_to_files") +@patch("moban.core.moban_factory.MobanEngine.render_to_files") def test_handle_targets_sequence(fake_renderer): from moban.mobanfile import handle_targets @@ -152,14 +45,14 @@ def test_handle_targets_sequence(fake_renderer): OUTPUT1 = "filterme.handlebars" # in the future, this could dynamic output OUTPUT2 = "filtered_output.txt" CONFIGURATION = "child.yaml" - TEMPLATE_DIRS = [os.path.join("tests", "fixtures", "mobanfile")] + TEMPLATE_DIRS = [fs.path.join("tests", "fixtures", "mobanfile")] DEFAULT_TEMPLATE_TYPE = "jinja2" options = dict( configuration=CONFIGURATION, template_type=DEFAULT_TEMPLATE_TYPE, template_dir=TEMPLATE_DIRS, - configuration_dir=os.path.join("tests", "fixtures"), + configuration_dir=fs.path.join("tests", "fixtures"), ) short_hand_targets = [{OUTPUT1: TEMPLATE1}, {OUTPUT2: OUTPUT1}] handle_targets(options, short_hand_targets) diff --git a/tests/mobanfile/test_targets.py b/tests/mobanfile/test_targets.py index 283ecf2c..dff5c850 100644 --- a/tests/mobanfile/test_targets.py +++ b/tests/mobanfile/test_targets.py @@ -1,6 +1,6 @@ -import os import uuid +import fs.path from nose.tools import eq_, raises from moban.mobanfile import targets from moban.exceptions import GroupTargetNotFound @@ -9,7 +9,7 @@ TEMPLATE = "a.jj2" OUTPUT = "a.output" CONFIGURATION = "data.config" -TEMPLATE_DIRS = [os.path.join("tests", "fixtures")] +TEMPLATE_DIRS = [fs.path.join("tests", "fixtures")] DEFAULT_TEMPLATE_TYPE = "default-template-type" diff --git a/tests/mobanfile/test_templates.py b/tests/mobanfile/test_templates.py index 21d90f2a..bd07f6ff 100644 --- a/tests/mobanfile/test_templates.py +++ b/tests/mobanfile/test_templates.py @@ -1,5 +1,4 @@ -import os - +import fs.path from mock import patch from nose.tools import eq_ from moban.mobanfile.templates import handle_template @@ -7,7 +6,7 @@ class TestHandleTemplateFunction: def setUp(self): - self.base_dir = [os.path.join("tests", "fixtures")] + self.base_dir = [fs.path.join("tests", "fixtures")] def test_copy_files(self): results = list( @@ -35,7 +34,7 @@ def test_listing_dir(self): expected = [ ( "copier-directory/level1-file1", - os.path.join("/tmp/copy-a-directory", "level1-file1"), + fs.path.join("/tmp/copy-a-directory", "level1-file1"), None, ) ] @@ -48,21 +47,21 @@ def test_listing_dir_recusively(self): ) expected = [ ( - os.path.join("copier-directory", "copier-sample-dir", "file1"), - os.path.join( + fs.path.join("copier-directory", "copier-sample-dir", "file1"), + fs.path.join( "/tmp/copy-a-directory", "copier-sample-dir", "file1" ), None, ), ( - os.path.join("copier-directory", "level1-file1"), - os.path.join("/tmp/copy-a-directory", "level1-file1"), + fs.path.join("copier-directory", "level1-file1"), + fs.path.join("/tmp/copy-a-directory", "level1-file1"), None, ), ] eq_( - sorted(expected, key=lambda x: x[0]), sorted(results, key=lambda x: x[0]), + sorted(expected, key=lambda x: x[0]), ) @patch("moban.reporter.report_error_message") diff --git a/tests/regression_tests/level-21-b-copy-templates-into-a-tar/.moban.yml b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/.moban.yml new file mode 100644 index 00000000..c9ceba44 --- /dev/null +++ b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/.moban.yml @@ -0,0 +1,19 @@ +configuration: + template_dir: + - "tar://template-sources.tar" +targets: + - output: "tar://my.tar!/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "tar://my.tar!/target_without_template_type" + template: file_extension_will_trigger.copy + - "tar://my.tar!/target_in_short_form": as_long_as_this_one_has.copy + - output: "tar://my.tar!/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "tar://my.tar!/test-dir" + template: dir-for-copying + template_type: copy + - output: "tar://my.tar!/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy diff --git a/tests/regression_tests/level-21-b-copy-templates-into-a-tar/README.rst b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/README.rst new file mode 100644 index 00000000..533269b5 --- /dev/null +++ b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/README.rst @@ -0,0 +1,83 @@ +Level 21-b: template copying from a tar to a tar +================================================================================ + +In level 15, with `.moban.yml`, you can copy templates to your destination. Now +with similiar moban syntax, let me show how to create a new zip file where +all templates are copied to. + +Explicit syntax:: + + targets: + - output: "tar://your.zip/explicit" + template: template_file + template_type: copy + + +Implicit syntax:: + + targets: + - output: "tar://your.zip/implicit" + template: template_file.copy + + +Shorthand syntax:: + + targets: + - "tar://your.zip/shorthand": template_file.copy + + +No implicit nor short hand syntax for the following directory copying unless +you take a look at `force-template-type`. When you read +`level-17-force-template-type-from-moban-file/README.rst`, you will find +out more. + + +Directory copying syntax:: + + + targets: + - output: "tar://your.zip/dest-dir" + template: source-dir + template_type: copy + + +Recursive directory copying syntax:: + + + targets: + - output: "tar://your.zip/dest-dir" + template: source-dir/** + template_type: copy + + +Evaluation +-------------------------------------------------------------------------------- + +Here is example moban file for copying:: + + configuration: + template_dir: + - "tar://template-sources.tar" + targets: + - output: "tar://my.tar/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "tar://my.tar/target_without_template_type" + template: file_extension_will_trigger.copy + - "tar://my.tar/target_in_short_form": as_long_as_this_one_has.copy + - output: "tar://my.tar/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "tar://my.tar/test-dir" + template: dir-for-copying + template_type: copy + - output: "tar://my.tar/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy + + +template copy does: + + +#. copies any template inside pre-declared template directory to anywhere. moban will create directory if needed. +#. copies any directory to anywhere. If "**" is followed, moban attempts to do recursive copying. diff --git a/tests/regression_tests/level-21-b-copy-templates-into-a-tar/template-sources.tar b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/template-sources.tar new file mode 100644 index 00000000..2f13227f Binary files /dev/null and b/tests/regression_tests/level-21-b-copy-templates-into-a-tar/template-sources.tar differ diff --git a/tests/regression_tests/level-21-c-copy-templates-from-a-tar/.moban.yml b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/.moban.yml new file mode 100644 index 00000000..d3b82590 --- /dev/null +++ b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/.moban.yml @@ -0,0 +1,19 @@ +configuration: + template_dir: + - "tar://template-sources.tar" +targets: + - output: "zip://my.zip!/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip!/target_without_template_type" + template: file_extension_will_trigger.copy + - "zip://my.zip!/target_in_short_form": as_long_as_this_one_has.copy + - output: "zip://my.zip!/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip!/test-dir" + template: dir-for-copying + template_type: copy + - output: "zip://my.zip!/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy diff --git a/tests/regression_tests/level-21-c-copy-templates-from-a-tar/README.rst b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/README.rst new file mode 100644 index 00000000..79ab39f2 --- /dev/null +++ b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/README.rst @@ -0,0 +1,24 @@ +Level 21-c: template copying from a tar to a zip +================================================================================ + +Here is another set of regression tests file for :: + + configuration: + template_dir: + - "tar://template-sources.tar" + targets: + - output: "zip://my.zip!/simple.file.copy" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip!/target_without_template_type" + template: file_extension_will_trigger.copy + - "zip://my.zip!/target_in_short_form": as_long_as_this_one_has.copy + - output: "zip://my.zip!/misc-1-copying/can-create-folder/if-not-exists.txt" + template: file-in-template-sources-folder.txt + template_type: copy + - output: "zip://my.zip!/test-dir" + template: dir-for-copying + template_type: copy + - output: "zip://my.zip!/test-recursive-dir" + template: dir-for-recusive-copying/** + template_type: copy diff --git a/tests/regression_tests/level-21-c-copy-templates-from-a-tar/template-sources.tar b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/template-sources.tar new file mode 100644 index 00000000..2f13227f Binary files /dev/null and b/tests/regression_tests/level-21-c-copy-templates-from-a-tar/template-sources.tar differ diff --git a/tests/requirements.txt b/tests/requirements.txt index a6b17478..055d0047 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -12,3 +12,5 @@ pypi-mobans-pkg arrow<0.14.0;python_version=="3.4" arrow;python_version!="3.4" jinja2_time +pypifs +gitfs2 diff --git a/tests/test_buffered_writer.py b/tests/test_buffered_writer.py new file mode 100644 index 00000000..e6684433 --- /dev/null +++ b/tests/test_buffered_writer.py @@ -0,0 +1,45 @@ +import os +import tempfile + +from moban import file_system +from nose.tools import eq_ +from moban.buffered_writer import BufferedWriter, write_file_out + +CONTENT = b""" + helloworld + + + + + """ +EXPECTED = "\n helloworld\n\n\n\n\n " + + +class TestBufferedWriter: + def setUp(self): + self.writer = BufferedWriter() + + def test_write_text(self): + test_file = "testout" + self.writer.write_file_out(test_file, CONTENT) + self.writer.close() + content = file_system.read_text(test_file) + eq_(content, EXPECTED) + os.unlink(test_file) + + def test_write_a_zip(self): + tmp_dir = os.path.normcase(tempfile.gettempdir()) + test_file = "zip://" + tmp_dir + "/testout.zip!/testout" + self.writer.write_file_out(test_file, CONTENT) + self.writer.close() + content = file_system.read_text(test_file) + eq_(content, EXPECTED) + os.unlink(os.path.join(tmp_dir, "testout.zip")) + + +def test_write_file_out(): + test_file = "testout" + write_file_out(test_file, CONTENT) + with open(test_file, "r") as f: + content = f.read() + eq_(content, EXPECTED) diff --git a/tests/test_copy_encoding.py b/tests/test_copy_encoding.py deleted file mode 100644 index de45bfbe..00000000 --- a/tests/test_copy_encoding.py +++ /dev/null @@ -1,20 +0,0 @@ -import os - -from moban.copy import ContentForwardEngine -from nose.tools import eq_ - - -class TestCopyEncoding: - def setUp(self): - template_path = os.path.join("tests", "fixtures") - self.engine = ContentForwardEngine([template_path]) - - def test_encoding_of_template(self): - template_content = self.engine.get_template("coala_color.svg") - with open("tests/fixtures/coala_color.svg", "rb") as expected: - expected = expected.read() - eq_(expected, template_content) - template_content = self.engine.get_template("non-unicode.char") - with open("tests/fixtures/non-unicode.char", "rb") as expected: - expected = expected.read() - eq_(expected, template_content) diff --git a/tests/test_copy_engine.py b/tests/test_copy_engine.py index eecf9a95..65a25262 100644 --- a/tests/test_copy_engine.py +++ b/tests/test_copy_engine.py @@ -1,5 +1,7 @@ import os +import fs.path +from moban import file_system from moban.copy import ContentForwardEngine from nose.tools import eq_ @@ -7,7 +9,8 @@ class TestContentForwardEngine: def setUp(self): template_path = os.path.join("tests", "fixtures") - self.engine = ContentForwardEngine([template_path]) + fs = file_system.get_multi_fs([template_path]) + self.engine = ContentForwardEngine(fs) def test_get_template(self): template_content = self.engine.get_template("copier-test01.csv") @@ -29,3 +32,20 @@ def test_apply_template(self): test_content = "simply forwarded" template_content = self.engine.apply_template(test_content, "not used") eq_(test_content, template_content) + + +class TestCopyEncoding: + def setUp(self): + template_path = fs.path.join("tests", "fixtures") + template_fs = file_system.get_multi_fs([template_path]) + self.engine = ContentForwardEngine(template_fs) + + def test_encoding_of_template(self): + template_content = self.engine.get_template("coala_color.svg") + with open("tests/fixtures/coala_color.svg", "rb") as expected: + expected = expected.read() + eq_(expected, template_content) + template_content = self.engine.get_template("non-unicode.char") + with open("tests/fixtures/non-unicode.char", "rb") as expected: + expected = expected.read() + eq_(expected, template_content) diff --git a/tests/test_definitions.py b/tests/test_definitions.py index 75d2bc60..98a78741 100644 --- a/tests/test_definitions.py +++ b/tests/test_definitions.py @@ -1,5 +1,6 @@ from nose.tools import eq_ -from moban.definitions import GitRequire, TemplateTarget +from moban.deprecated import GitRequire +from moban.definitions import TemplateTarget def test_git_require_repr(): diff --git a/tests/test_docs.py b/tests/test_docs.py index 7f78cb30..2c711357 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,44 +1,36 @@ import os -import sys -from textwrap import dedent -from mock import patch -from moban.main import main from nose.tools import eq_ +from .utils import Docs, custom_dedent -def custom_dedent(long_texts): - refined = dedent(long_texts) - if refined.startswith("\n"): - refined = refined[1:] - return refined - - -class TestTutorial: - def setUp(self): - self.current = os.getcwd() +class TestTutorial(Docs): def test_level_1(self): expected = "world" folder = "level-1-jinja2-cli" self._moban(folder, expected) + def test_level_1_custom_define(self): + expected = "maailman" + folder = "level-1-jinja2-cli" + args = ["moban", "-d", "hello=maailman", "-t", "a.template"] + self.run_moban(args, folder, [("moban.output", expected)]) + def test_level_2(self): - expected = custom_dedent( - """ + expected = """ ========header============ world ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-2-template-inheritance" self._moban(folder, expected) def test_level_3(self): - expected = custom_dedent( - """ + expected = """ ========header============ world @@ -47,13 +39,12 @@ def test_level_3(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-3-data-override" self._moban(folder, expected) def test_level_4(self): - expected = custom_dedent( - """ + expected = """ ========header============ world @@ -62,13 +53,12 @@ def test_level_4(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-4-single-command" - self._raw_moban(["moban"], folder, expected, "a.output") + self.run_moban(["moban"], folder, [("a.output", expected)]) def test_level_5(self): - expected = custom_dedent( - """ + expected = """ ========header============ world @@ -79,13 +69,12 @@ def test_level_5(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-5-custom-configuration" - self._raw_moban(["moban"], folder, expected, "a.output") + self.run_moban(["moban"], folder, [("a.output", expected)]) def test_level_6(self): - expected = custom_dedent( - """ + expected = """ ========header============ world2 @@ -96,68 +85,93 @@ def test_level_6(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-6-complex-configuration" - self._raw_moban(["moban"], folder, expected, "a.output2") + self.run_moban(["moban"], folder, [("a.output2", expected)]) + + def test_level_20(self): + expected = """ + ========header============ + + world2 + + shijie + + this demonstrates jinja2's include statement + + ========footer============ + """ + expected = custom_dedent(expected) + folder = "level-20-templates-configs-in-zip-or-tar" + self.run_moban_with_fs( + ["moban"], folder, [("zip://a.zip!/a.output2", expected)] + ) def test_level_7(self): - expected = custom_dedent( - """ + expected = """ Hello, you are in level 7 example Hello, you are not in level 7 """ - ) + expected = custom_dedent(expected) + folder = "level-7-use-custom-jinja2-filter-test-n-global" - self._raw_moban(["moban"], folder, expected, "test.output") + self.run_moban(["moban"], folder, [("test.output", expected)]) def test_level_8(self): expected = "it is a test\n" folder = "level-8-pass-a-folder-full-of-templates" check_file = os.path.join("templated-folder", "my") - self._raw_moban(["moban"], folder, expected, check_file) + self.run_moban(["moban"], folder, [(check_file, expected)]) def test_level_9(self): expected = "pypi-mobans: moban dependency as pypi package" folder = "level-9-moban-dependency-as-pypi-package" - self._raw_moban(["moban"], folder, expected, "test.txt") + self.run_moban(["moban"], folder, [("test.txt", expected)]) + + def test_level_9_deprecated(self): + expected = "pypi-mobans: moban dependency as pypi package" + folder = "deprecated-level-9-moban-dependency-as-pypi-package" + self.run_moban(["moban"], folder, [("test.txt", expected)]) def test_level_10(self): expected = "pypi-mobans: moban dependency as git repo" folder = "level-10-moban-dependency-as-git-repo" - self._raw_moban(["moban"], folder, expected, "test.txt") + self.run_moban(["moban"], folder, [("test.txt", expected)]) + + def test_level_10_deprecated(self): + expected = "pypi-mobans: moban dependency as git repo" + folder = "deprecated-level-10-moban-dependency-as-git-repo" + self.run_moban(["moban"], folder, [("test.txt", expected)]) def test_level_11(self): expected = "handlebars does not support inheritance\n" folder = "level-11-use-handlebars" - self._raw_moban(["moban"], folder, expected, "a.output") + self.run_moban(["moban"], folder, [("a.output", expected)]) - def test_level_12_a(self): - expected_a = custom_dedent( - """ + def test_level_12(self): + expected_a = """ world world world world """ - ) - folder = "level-12-use-template-engine-extensions" - self._raw_moban(["moban"], folder, expected_a, "a.output") - - def test_level_12_b(self): - expected_b = custom_dedent( - """ + expected_b = """ 142 42 142 """ - ) + expected_a = custom_dedent(expected_a) + expected_b = custom_dedent(expected_b) folder = "level-12-use-template-engine-extensions" - self._raw_moban(["moban"], folder, expected_b, "b.output") + self.run_moban( + ["moban"], + folder, + [("a.output", expected_a), ("b.output", expected_b)], + ) def test_level_13_json(self): - expected = custom_dedent( - """ + expected = """ ========header============ world from child.json @@ -166,14 +180,13 @@ def test_level_13_json(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-13-any-data-override-any-data" commands = ["moban", "-c", "child.json", "-t", "a.template"] - self._raw_moban(commands, folder, expected, "moban.output") + self.run_moban(commands, folder, [("moban.output", expected)]) def test_level_13_yaml(self): - expected = custom_dedent( - """ + expected = """ ========header============ world from child.yaml @@ -182,14 +195,13 @@ def test_level_13_yaml(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-13-any-data-override-any-data" commands = ["moban", "-c", "child.yaml", "-t", "a.template"] - self._raw_moban(commands, folder, expected, "moban.output") + self.run_moban(commands, folder, [("moban.output", expected)]) def test_level_14_custom(self): - expected = custom_dedent( - """ + expected = """ ========header============ world from child.cusom @@ -198,41 +210,70 @@ def test_level_14_custom(self): ========footer============ """ - ) + expected = custom_dedent(expected) folder = "level-14-custom-data-loader" commands = ["moban"] - self._raw_moban(commands, folder, expected, "a.output") + self.run_moban(commands, folder, [("a.output", expected)]) def test_level_15_copy_templates_as_target(self): expected = "test file\n" folder = "level-15-copy-templates-as-target" - self._raw_moban(["moban"], folder, expected, "simple.file") - - _verify_content( - "target_without_template_type", - "file extension will trigger copy engine\n", - ) - _verify_content( - "target_in_short_form", + assertions = [ + ("simple.file", expected), + ( + "target_without_template_type", + "file extension will trigger copy engine\n", + ), ( - "it is OK to have a short form, " - + "but the file to be 'copied' shall have 'copy' extension, " - + "so as to trigger ContentForwardEngine, 'copy' engine.\n" + "target_in_short_form", + ( + "it is OK to have a short form, " + + "but the file to be 'copied' shall have 'copy' extension, " + + "so as to trigger ContentForwardEngine, 'copy' engine.\n" + ), ), + ] + self.run_moban(["moban"], folder, assertions) + + def test_level_21_copy_templates_into_zips(self): + expected = "test file\n" + + folder = "level-21-copy-templates-into-an-alien-file-system" + long_url = ( + "zip://my.zip!/test-recursive-dir/sub_directory_is_copied" + + "/because_star_star_is_specified.txt" ) + criterias = [ + ["zip://my.zip!/simple.file", expected], + [ + "zip://my.zip!/target_without_template_type", + "file extension will trigger copy engine\n", + ], + [ + "zip://my.zip!/target_in_short_form", + ( + "it is OK to have a short form, " + + "but the file to be 'copied' shall have 'copy' extension, " + + "so as to trigger ContentForwardEngine, 'copy' engine.\n" + ), + ], + ["zip://my.zip!/test-dir/afile.txt", "dir for copying\n"], + [long_url, "dest_directory: source_directory/**\n"], + ] + self.run_moban_with_fs(["moban"], folder, criterias) def test_level_16_group_targets_using_template_type(self): expected = "test file\n" folder = "level-16-group-targets-using-template-type" - self._raw_moban(["moban"], folder, expected, "simple.file") + self.run_moban(["moban"], folder, [("simple.file", expected)]) def test_level_17_force_template_type_from_moban_file(self): expected = "test file\n" folder = "level-17-force-template-type-from-moban-file" - self._raw_moban(["moban"], folder, expected, "simple.file") + self.run_moban(["moban"], folder, [("simple.file", expected)]) def test_level_18_user_defined_template_types(self): from datetime import datetime @@ -240,24 +281,28 @@ def test_level_18_user_defined_template_types(self): expected = "{date}\n".format(date=datetime.now().strftime("%Y-%m-%d")) folder = "level-18-user-defined-template-types" - self._raw_moban(["moban"], folder, expected, "a.output") - - _verify_content("b.output", "shijie\n") + self.run_moban( + ["moban"], + folder, + [("a.output", expected), ("b.output", "shijie\n")], + ) def test_level_19_without_group_target(self): expected = "test file\n" folder = "level-19-moban-a-sub-group-in-targets" - self._raw_moban(["moban"], folder, expected, "simple.file") - _verify_content("a.output", "I will not be selected in level 19\n") - os.unlink("a.output") + assertions = [ + ("simple.file", expected), + ("a.output", "I will not be selected in level 19\n"), + ] + self.run_moban(["moban"], folder, assertions) def test_level_19_with_group_target(self): expected = "test file\n" folder = "level-19-moban-a-sub-group-in-targets" - self._raw_moban( - ["moban", "-g", "copy"], folder, expected, "simple.file" + self.run_moban( + ["moban", "-g", "copy"], folder, [("simple.file", expected)] ) # make sure only copy target is executed eq_(False, os.path.exists("a.output")) @@ -266,26 +311,8 @@ def test_misc_1(self): expected = "test file\n" folder = "misc-1-copying-templates" - self._raw_moban(["moban"], folder, expected, "simple.file") + self.run_moban(["moban"], folder, [("simple.file", expected)]) def _moban(self, folder, expected): args = ["moban", "-c", "data.yml", "-t", "a.template"] - self._raw_moban(args, folder, expected, "moban.output") - - def _raw_moban(self, args, folder, expected, output): - os.chdir(os.path.join("docs", folder)) - with patch.object(sys, "argv", args): - main() - _verify_content(output, expected) - os.unlink(output) - - def tearDown(self): - if os.path.exists(".moban.hashes"): - os.unlink(".moban.hashes") - os.chdir(self.current) - - -def _verify_content(file_name, expected): - with open(file_name, "r") as f: - content = f.read() - eq_(content, expected) + self.run_moban(args, folder, [("moban.output", expected)]) diff --git a/tests/test_file_system.py b/tests/test_file_system.py new file mode 100644 index 00000000..5b2eb26b --- /dev/null +++ b/tests/test_file_system.py @@ -0,0 +1,245 @@ +import os +import sys +import stat + +from nose import SkipTest +from moban import file_system +from nose.tools import eq_, raises +from moban.exceptions import FileNotFound + +LOCAL_FOLDER = "tests/fixtures" +LOCAL_FILE = LOCAL_FOLDER + "/a.jj2" +ZIP_FILE = "zip://tests/fixtures/file_system/template-sources.zip" +TAR_FILE = "tar://tests/fixtures/file_system/template-sources.tar" + +FILE = "file-in-template-sources-folder.txt" + +ZIP_URL = ZIP_FILE + "!/" + FILE +TAR_URL = TAR_FILE + "!/" + FILE + +TEST_SPECS = [LOCAL_FILE, ZIP_URL, TAR_URL] + + +TEST_FS_SPECS = [LOCAL_FOLDER, ZIP_FILE, TAR_FILE] + + +def test_open_file(): + for url in TEST_SPECS: + with file_system.open_file(url): + pass + + +def test_open_fs(): + for url in TEST_FS_SPECS: + with file_system.open_fs(url): + pass + + +TEST_FILE_CONTENT_SPECS = [ + [LOCAL_FILE, "{{key}} {{pass}}"], + [ZIP_URL, "test file\n"], + [TAR_URL, "test file\n"], +] + + +def test_read_unicode(): + for url, expected in TEST_FILE_CONTENT_SPECS: + content = file_system.read_unicode(url) + eq_(content, expected) + + +TEST_FILE_CONTENT_SPECS_BINARY = [ + [LOCAL_FILE, b"{{key}} {{pass}}"], + [ZIP_URL, b"test file\n"], + [TAR_URL, b"test file\n"], +] + + +def test_read_binary(): + for url, expected in TEST_FILE_CONTENT_SPECS_BINARY: + content = file_system.read_binary(url) + eq_(content, expected) + + +TEST_WRITE_BYTES_SPEC = [ + ["test.binary", b"abc"], + ["zip://test.zip!/test.binary", b"abc"], + ["tar://test.tar!/test.binary", b"abc"], +] + + +def test_write_bytes(): + for url, content in TEST_WRITE_BYTES_SPEC: + file_system.write_bytes(url, content) + + for url, expected in TEST_WRITE_BYTES_SPEC: + content = file_system.read_bytes(url) + eq_(content, expected) + + for file_name in ["test.binary", "test.zip", "test.tar"]: + os.unlink(file_name) + + +TEST_DIR_SPEC = [ + [LOCAL_FOLDER, True], + [ZIP_FILE + "!/dir-for-copying", True], + [TAR_FILE + "!/dir-for-copying", True], + [ZIP_URL, False], + [TAR_URL, False], + [LOCAL_FILE, False], +] + + +def test_is_dir(): + for url, expected in TEST_DIR_SPEC: + status = file_system.is_dir(url) + eq_(status, expected) + + +def test_is_file(): + for url, is_dir in TEST_DIR_SPEC: + status = file_system.is_file(url) + expected = not is_dir + eq_(status, expected) + + +TEST_URL_EXITENCE_SPEC = [ + [LOCAL_FOLDER, True], + [ZIP_FILE + "!/dir-for-copying", True], + [TAR_FILE + "!/dir-for-copying", True], + [ZIP_URL, True], + [TAR_URL, True], + [LOCAL_FILE, True], + # ['zip://abc.zip', False], + # ['tar://abc.tar', False], bug with fs.zipfs. raise it later + ["abcx", False], +] + + +def test_exists(): + for url, expected in TEST_URL_EXITENCE_SPEC: + status = file_system.exists(url) + eq_(status, expected) + + +TEST_LIST_DIR_SPEC = [ + [ + LOCAL_FOLDER + "/file_system", + ["template-sources.zip", "template-sources.tar"], + ], + [ + ZIP_FILE + "!/dir-for-copying", + ["afile.txt", "sub_directory_is_not_copied"], + ], + [ + TAR_FILE + "!/dir-for-copying", + ["afile.txt", "sub_directory_is_not_copied"], + ], +] + + +def test_list_dir(): + for url, expected in TEST_LIST_DIR_SPEC: + file_list = sorted(list(file_system.list_dir(url))) + eq_(file_list, sorted(expected)) + + +TEST_FILE_PATH = [ + [ + LOCAL_FOLDER + "/file_system", + os.path.normpath( + os.path.join(os.getcwd(), "tests/fixtures/file_system") + ), + ] +] + + +def test_abspath(): + for path, expected in TEST_FILE_PATH: + url = file_system.abspath(path) + eq_(url, expected) + + +TEST_FILE_URL = [ + [ + LOCAL_FOLDER + "/file_system", + "osfs://" + + os.path.normpath( + os.path.join(os.getcwd(), "tests/fixtures/file_system") + ), + ] +] + + +def test_fs_url(): + for path, expected in TEST_FILE_URL: + url = file_system.fs_url(path) + eq_(url, expected.replace("\\", "/")) + + +URL_JOIN_TEST_FIXTURES = [ + ["parent", "child", "parent/child"], + ["zip://test.zip", "file", "zip://test.zip!/file"], + ["/root", "path", "/root/path"], +] + + +def test_url_join(): + for parent, child, expected_path in URL_JOIN_TEST_FIXTURES: + actual = file_system.url_join(parent, child) + eq_(actual, expected_path) + + +def create_file(test_file, permission): + with open(test_file, "w") as f: + f.write("test") + + os.chmod(test_file, permission) + + +def test_file_permission_copy(): + if sys.platform == "win32": + raise SkipTest("No actual chmod on windows") + test_source = "test_file_permission_copy1" + test_dest = "test_file_permission_copy2" + create_file(test_source, 0o755) + create_file(test_dest, 0o646) + file_system.file_permissions_copy(test_source, test_dest) + eq_( + stat.S_IMODE(os.lstat(test_source).st_mode), + stat.S_IMODE(os.lstat(test_dest).st_mode), + ) + os.unlink(test_source) + os.unlink(test_dest) + + +def file_permissions_disabled_on_windows(): + if sys.platform == "win32": + permissions = file_system.file_permissions("abc") + eq_("no-permission-support", permissions) + else: + raise SkipTest("No test required") + + +@raises(FileNotFound) +def test_file_permissions_file_not_found(): + file_system.file_permissions("I does not exist") + + +def test_file_permission_copy_symlink(): + if sys.platform == "win32": + raise SkipTest("No symlink on windows") + test_source = "test_file_permission_copy1" + test_dest = "test_file_permission_copy2" + test_symlink = "test_file_permission_symlink" + create_file(test_source, 0o046) + os.symlink(test_source, test_symlink) + create_file(test_dest, 0o646) + file_system.file_permissions_copy(test_source, test_dest) + eq_( + stat.S_IMODE(os.lstat(test_source).st_mode), + stat.S_IMODE(os.lstat(test_dest).st_mode), + ) + os.unlink(test_source) + os.unlink(test_dest) + os.unlink(test_symlink) diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 2e22ef17..3a8d5ae9 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -2,12 +2,15 @@ import sys from nose import SkipTest +from moban import file_system from moban.hashstore import HashStore class TestHashStore: def setUp(self): - self.source_template = os.path.join("tests", "fixtures", "a.jj2") + self.source_template = file_system.path_join( + "tests", "fixtures", "a.jj2" + ) self.fixture = ( "test.out", "test content".encode("utf-8"), diff --git a/tests/test_main.py b/tests/test_main.py index 3f8b27db..96adba08 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -4,7 +4,7 @@ import moban.exceptions as exceptions from mock import patch -from nose.tools import raises, assert_raises +from nose.tools import eq_, raises, assert_raises class TestException: @@ -123,7 +123,7 @@ def test_double_underscore_main( class TestExitCodes: @raises(SystemExit) @patch("moban.main.handle_moban_file") - @patch("moban.mobanfile.find_default_moban_file") + @patch("moban.main.find_default_moban_file") def test_has_many_files_with_exit_code( self, fake_find_file, fake_moban_file ): @@ -136,7 +136,7 @@ def test_has_many_files_with_exit_code( @raises(SystemExit) @patch("moban.main.handle_command_line") - @patch("moban.mobanfile.find_default_moban_file") + @patch("moban.main.find_default_moban_file") def test_handle_single_change_with_exit_code( self, fake_find_file, fake_command_line ): @@ -148,7 +148,7 @@ def test_handle_single_change_with_exit_code( main() @patch("moban.main.handle_moban_file") - @patch("moban.mobanfile.find_default_moban_file") + @patch("moban.main.find_default_moban_file") def test_has_many_files(self, fake_find_file, fake_moban_file): fake_find_file.return_value = "abc" fake_moban_file.return_value = 1 @@ -158,7 +158,7 @@ def test_has_many_files(self, fake_find_file, fake_moban_file): main() @patch("moban.main.handle_command_line") - @patch("moban.mobanfile.find_default_moban_file") + @patch("moban.main.find_default_moban_file") def test_handle_single_change(self, fake_find_file, fake_command_line): fake_find_file.return_value = None fake_command_line.return_value = 1 @@ -166,3 +166,35 @@ def test_handle_single_change(self, fake_find_file, fake_command_line): with patch.object(sys, "argv", ["moban"]): main() + + +class TestFinder: + def setUp(self): + self.patcher = patch("moban.file_system.exists") + self.fake_file_existence = self.patcher.start() + self.fake_file_existence.__name__ = "fake" + self.fake_file_existence.__module__ = "fake" + + def tearDown(self): + self.patcher.stop() + + def test_moban_yml(self): + self.fake_file_existence.return_value = True + from moban.main import find_default_moban_file + + actual = find_default_moban_file() + eq_(".moban.yml", actual) + + def test_moban_yaml(self): + self.fake_file_existence.side_effect = [False, True] + from moban.main import find_default_moban_file + + actual = find_default_moban_file() + eq_(".moban.yaml", actual) + + def test_no_moban_file(self): + self.fake_file_existence.side_effect = [False, False] + from moban.main import find_default_moban_file + + actual = find_default_moban_file() + assert actual is None diff --git a/tests/test_regression.py b/tests/test_regression.py index f80e8336..85ab9a48 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -1,23 +1,18 @@ import os import sys import filecmp -from textwrap import dedent +import fs.path from mock import patch from moban.main import main -from nose.tools import eq_ +from .utils import Docs -def custom_dedent(long_texts): - refined = dedent(long_texts) - if refined.startswith("\n"): - refined = refined[1:] - return refined - -class TestRegression: +class TestRegression(Docs): def setUp(self): - self.current = os.getcwd() + super(TestRegression, self).setUp() + self.base_folder = fs.path.join("tests", "regression_tests") def test_coping_binary_file(self): folder = "regr-01-copy-binary-file" @@ -25,26 +20,69 @@ def test_coping_binary_file(self): self._raw_moban( args, folder, - os.path.join("copy-source", "image.png"), + fs.path.join("copy-source", "image.png"), "regression-test.png", ) + def test_level_21_copy_templates_into_tars(self): + expected = "test file\n" + + folder = "level-21-b-copy-templates-into-a-tar" + long_url = ( + "tar://my.tar!/test-recursive-dir/sub_directory_is_copied" + + "/because_star_star_is_specified.txt" + ) + criterias = [ + ["tar://my.tar!/simple.file", expected], + [ + "tar://my.tar!/target_without_template_type", + "file extension will trigger copy engine\n", + ], + [ + "tar://my.tar!/target_in_short_form", + ( + "it is OK to have a short form, " + + "but the file to be 'copied' shall have 'copy' extension, " + + "so as to trigger ContentForwardEngine, 'copy' engine.\n" + ), + ], + ["tar://my.tar!/test-dir/afile.txt", "dir for copying\n"], + [long_url, "dest_directory: source_directory/**\n"], + ] + self.run_moban_with_fs(["moban"], folder, criterias) + + def test_level_21_copy_templates_from_tars(self): + expected = "test file\n" + + folder = "level-21-c-copy-templates-from-a-tar" + long_url = ( + "zip://my.zip!/test-recursive-dir/sub_directory_is_copied" + + "/because_star_star_is_specified.txt" + ) + criterias = [ + ["zip://my.zip!/simple.file", expected], + [ + "zip://my.zip!/target_without_template_type", + "file extension will trigger copy engine\n", + ], + [ + "zip://my.zip!/target_in_short_form", + ( + "it is OK to have a short form, " + + "but the file to be 'copied' shall have 'copy' extension, " + + "so as to trigger ContentForwardEngine, 'copy' engine.\n" + ), + ], + ["zip://my.zip!/test-dir/afile.txt", "dir for copying\n"], + [long_url, "dest_directory: source_directory/**\n"], + ] + self.run_moban_with_fs(["moban"], folder, criterias) + def _raw_moban(self, args, folder, expected, output): - base_dir = os.path.join("tests", "regression_tests") - os.chdir(os.path.join(base_dir, folder)) + base_dir = fs.path.join("tests", "regression_tests") + os.chdir(fs.path.join(base_dir, folder)) with patch.object(sys, "argv", args): main() status = filecmp.cmp(output, expected) os.unlink(output) assert status - - def tearDown(self): - if os.path.exists(".moban.hashes"): - os.unlink(".moban.hashes") - os.chdir(self.current) - - -def _verify_content(file_name, expected): - with open(file_name, "r") as f: - content = f.read() - eq_(content, expected) diff --git a/tests/test_utils.py b/tests/test_utils.py index 7ce9ffcf..ab395fe6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,89 +1,8 @@ import os -import sys -import stat from shutil import rmtree from mock import patch -from nose import SkipTest -from nose.tools import eq_, raises -from moban.utils import ( - mkdir_p, - write_file_out, - file_permissions, - get_template_path, - file_permissions_copy, -) -from moban.exceptions import FileNotFound - - -def create_file(test_file, permission): - with open(test_file, "w") as f: - f.write("test") - - os.chmod(test_file, permission) - - -def test_file_permission_copy(): - if sys.platform == "win32": - raise SkipTest("No actual chmod on windows") - test_source = "test_file_permission_copy1" - test_dest = "test_file_permission_copy2" - create_file(test_source, 0o046) - create_file(test_dest, 0o646) - file_permissions_copy(test_source, test_dest) - eq_( - stat.S_IMODE(os.lstat(test_source).st_mode), - stat.S_IMODE(os.lstat(test_dest).st_mode), - ) - os.unlink(test_source) - os.unlink(test_dest) - - -def file_permissions_disabled_on_windows(): - if sys.platform == "win32": - permissions = file_permissions("abc") - eq_("no-permission-support", permissions) - else: - raise SkipTest("No test required") - - -@raises(FileNotFound) -def test_file_permissions_file_not_found(): - file_permissions("I does not exist") - - -def test_file_permission_copy_symlink(): - if sys.platform == "win32": - raise SkipTest("No symlink on windows") - test_source = "test_file_permission_copy1" - test_dest = "test_file_permission_copy2" - test_symlink = "test_file_permission_symlink" - create_file(test_source, 0o046) - os.symlink(test_source, test_symlink) - create_file(test_dest, 0o646) - file_permissions_copy(test_source, test_dest) - eq_( - stat.S_IMODE(os.lstat(test_source).st_mode), - stat.S_IMODE(os.lstat(test_dest).st_mode), - ) - os.unlink(test_source) - os.unlink(test_dest) - os.unlink(test_symlink) - - -def test_write_file_out(): - content = b""" - helloworld - - - - - """ - test_file = "testout" - write_file_out(test_file, content) - with open(test_file, "r") as f: - content = f.read() - eq_(content, "\n helloworld\n\n\n\n\n ") +from moban.utils import mkdir_p def test_mkdir_p(): @@ -93,35 +12,10 @@ def test_mkdir_p(): rmtree(test_path) -@raises(FileNotFound) -def test_get_template_path_with_error(): - temp_dirs = [ - os.path.join("tests", "fixtures", "template-tests"), - os.path.join("tests", "abc"), - ] - template = "I-do-not-exist.jj2" - get_template_path(temp_dirs, template) - - -def test_get_template_path(): - temp_dirs = [ - os.path.join("tests", "fixtures", "template-tests"), - os.path.join("tests", "abc"), - os.path.join("tests", "abc"), - ] - template = "a.jj2" - template_path = get_template_path(temp_dirs, template) - expected = os.path.join( - os.getcwd(), - os.path.join("tests", "fixtures", "template-tests", "a.jj2"), - ) - eq_(template_path, expected) - - @patch("subprocess.check_call") def test_pip_install(fake_check_all): import sys - from moban.utils import pip_install + from moban.deprecated import pip_install pip_install(["package1", "package2"]) fake_check_all.assert_called_with( diff --git a/tests/test_yaml_loader.py b/tests/test_yaml_loader.py deleted file mode 100644 index fe3cefe5..00000000 --- a/tests/test_yaml_loader.py +++ /dev/null @@ -1,35 +0,0 @@ -import os - -from nose.tools import eq_, raises -from moban.data_loaders.yaml import open_yaml -from moban.data_loaders.manager import load_data - - -def test_simple_yaml(): - test_file = os.path.join("tests", "fixtures", "simple.yaml") - data = open_yaml(test_file) - eq_(data, {"simple": "yaml"}) - - -def test_inheritance_yaml(): - test_file = os.path.join("tests", "fixtures", "child.yaml") - data = load_data(os.path.join("tests", "fixtures", "config"), test_file) - eq_(data, {"key": "hello world", "pass": "ox"}) - - -@raises(IOError) -def test_exception(): - test_file = os.path.join("tests", "fixtures", "orphan.yaml") - load_data(os.path.join("tests", "fixtures", "config"), test_file) - - -@raises(IOError) -def test_exception_2(): - test_file = os.path.join("tests", "fixtures", "dragon.yaml") - load_data(os.path.join("tests", "fixtures", "config"), test_file) - - -@raises(IOError) -def test_exception_3(): - test_file = os.path.join("tests", "fixtures", "dragon.yaml") - load_data(None, test_file) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..986951ed --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,64 @@ +import os +import sys +from textwrap import dedent + +from mock import patch +from moban import file_system +from moban.main import main +from nose.tools import eq_ +from fs.opener.parse import parse_fs_url + + +def verify_content(file_name, expected): + with open(file_name, "r") as f: + content = f.read() + eq_(content, expected) + + +def verify_content_with_fs(file_name, expected): + content = file_system.read_unicode(file_name) + eq_(content, expected) + + +def run_moban(args, folder, criterias): + with patch.object(sys, "argv", args): + main() + for output, expected in criterias: + verify_content(output, expected) + os.unlink(output) + + +def run_moban_with_fs(args, folder, criterias): + with patch.object(sys, "argv", args): + main() + + for output, expected in criterias: + verify_content_with_fs(output, expected) + result = parse_fs_url(output) + os.unlink(result.resource) # delete the zip file + + +class Docs(object): + def setUp(self): + self.current = os.getcwd() + self.base_folder = "docs" + + def tearDown(self): + if os.path.exists(".moban.hashes"): + os.unlink(".moban.hashes") + os.chdir(self.current) + + def run_moban(self, moban_cli, working_directory, assertions): + os.chdir(os.path.join(self.base_folder, working_directory)) + run_moban(moban_cli, None, assertions) + + def run_moban_with_fs(self, moban_cli, working_directory, assertions): + os.chdir(os.path.join(self.base_folder, working_directory)) + run_moban_with_fs(moban_cli, None, assertions) + + +def custom_dedent(long_texts): + refined = dedent(long_texts) + if refined.startswith("\n"): + refined = refined[1:] + return refined