RHELPLAN-42920 - Collections - script - add tox/travis tests for coll…

…ection conversion, verification

Adding collection to envlist and travis in tox.ini.
Adding script .travis/runcollection.sh that is invoked by "tox -e collection".

yamllint: since collection conversion removes some line breaks,
          yamllint tends to fail with line too long. Setting the
          level to warning in .yamllint_defaults.yml.

utils.sh: supporting collections format path, e.g.,
  - "$TOPDIR"/tests/ROLENAME/setup_module_utils.sh
  - SRC_COLL_MODULE_UTILS_DIR and DEST_COLL_MODULE_UTILS_DIR

.travis/runcollection.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+
+set -e
+
+#uncomment if you use $ME - otherwise set in utils.sh
+#ME=$(basename "$0")
+SCRIPTDIR=$(readlink -f "$(dirname "$0")")
+
+. "${SCRIPTDIR}/utils.sh"
+. "${SCRIPTDIR}/config.sh"
+
+# Collection commands that are run when `tox -e collection`:
+role=$(basename "${TOPDIR}")
+toxworkdir=${1:-${TOPDIR}/.tox}
+cd "${toxworkdir}"
+toxworkdir=$(pwd)
+envlist=${2:- "black,pylint,flake8,yamllint,py38,shellcheck"}
+
+git clone https://github.com/linux-system-roles/auto-maintenance
+
+python auto-maintenance/lsr_role2collection.py --src-path "${TOPDIR}/.." --dest-path "${toxworkdir}" --role "${role}" > "${toxworkdir}"/collection.out 2>&1
+
+cd ansible_collections/fedora/system_roles
+tox -e "${envlist}" 2>&1 | tee "${toxworkdir}"/collection.tox.out || :
+
+rm -rf "${toxworkdir}"/auto-maintenance "${toxworkdir}"/ansible_collections
+cd "${TOPDIR}"
+res=$(grep "^ERROR: .*failed" "${toxworkdir}"/collection.tox.out || :)
+if [ "$res" != "" ]; then
+    lsr_error "${ME}: tox in the converted collection format failed."
+    exit 1
+fi

.travis/utils.sh

@@ -172,6 +172,22 @@ function lsr_venv_python_matches_system_python() {
   lsr_compare_pythons "${1:-python}" -eq "$syspython"
 }
 
+function run_setup_module_utils() {
+  local srcdir
+  srcdir="$1"
+  local destdir
+  destdir="$2"
+  if [ -x "$TOPDIR/tests/setup_module_utils.sh" ]; then
+    bash "$TOPDIR/tests/setup_module_utils.sh" "$srcdir" "$destdir"
+  else
+    setup_module_utils=$( ls "$TOPDIR"/tests/*/setup_module_utils.sh )
+    setup_module_util=$( echo "$setup_module_utils" | awk '{print $1}' )
+    if [ "$setup_module_util" != "" ]; then
+      bash "$setup_module_util" "$srcdir" "$destdir"
+    fi
+  fi
+}
+
 ##
 # lsr_setup_module_utils [$1] [$2]
 #
@@ -189,7 +205,16 @@ function lsr_setup_module_utils() {
   local destdir
   destdir="${2:-$DEST_MODULE_UTILS_DIR}"
   if [ -n "$srcdir" ] && [ -d "$srcdir" ] && [ -n "$destdir" ] && [ -d "$destdir" ]; then
-    bash "$TOPDIR/tests/setup_module_utils.sh" "$srcdir" "$destdir"
+    run_setup_module_utils "$srcdir" "$destdir"
+  else
+    srcdir="${SRC_COLL_MODULE_UTILS_DIR}"
+    destdir="${DEST_COLL_MODULE_UTILS_DIR}"
+    if [ -n "$destdir" ] && [ -d "$destdir" ]; then
+      if [ -n "$srcdir" ] && [ ! -d "$srcdir" ]; then
+        mkdir -p "$srcdir"
+      fi
+      run_setup_module_utils "$srcdir" "$destdir"
+    fi
   fi
 }
 

.yamllint_defaults.yml

@@ -13,4 +13,6 @@ rules:
   truthy:
     allowed-values: ["yes", "no", "true", "false"]
     level: error
+  line-length:
+    level: warning
   document-start: disable

tox.ini

@@ -3,7 +3,7 @@
 envlist =
     black, pylint, flake8, yamllint
     py{26,27,36,37,38}, shellcheck
-    custom
+    collection, custom
 skipsdist = true
 skip_missing_interpreters = true
 
@@ -13,10 +13,13 @@ basepython = python3
 changedir = {toxinidir}/tests
 # List common dependencies for Python interpreters here:
 setenv =
-    PYTHONPATH = {toxinidir}/library:{toxinidir}/module_utils
+    PYTHONPATH = {envsitepackagesdir}:{toxinidir}/library:{toxinidir}/module_utils:{toxinidir}/plugins/modules:{toxinidir}/plugins/module_utils
     LC_ALL = C.UTF-8
     SRC_MODULE_UTILS_DIR = {envsitepackagesdir}/ansible/module_utils
     DEST_MODULE_UTILS_DIR = {toxinidir}/module_utils
+    SRC_COLL_MODULE_UTILS_DIR = {envsitepackagesdir}/ansible_collections/fedora/system_roles/plugins/module_utils
+    DEST_COLL_MODULE_UTILS_DIR = {toxinidir}/plugins/module_utils
+    RUN_PYLINT_SETUP_MODULE_UTILS = true
 deps =
     py{26,27,36,37,38}: pytest-cov
     py{27,36,37,38}: pytest>=3.5.1
@@ -70,8 +73,8 @@ commands =
     bash {toxinidir}/.travis/runblack.sh --check --diff .
 
 [testenv:pylint]
-envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:2.7}
-basepython = python2.7
+envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:3.6}
+basepython = python3.6
 passenv = RUN_PYLINT_*
 changedir = {toxinidir}
 deps =
@@ -164,6 +167,16 @@ commands =
     {[testenv:molecule_syntax]commands}
     {[testenv:molecule_test]commands}
 
+[testenv:collection]
+changedir = {toxinidir}
+ansible_python_interpreter = /usr/bin/python3
+deps =
+    ruamel.yaml
+    ansible
+    jmespath
+commands =
+    bash {toxinidir}/.travis/runcollection.sh {toxworkdir}
+
 [testenv:custom]
 changedir = {toxinidir}
 envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:custom}
@@ -184,7 +197,7 @@ addopts = -rxs
 [flake8]
 show_source = true
 max-line-length = 88
-ignore = E402,W503
+ignore = E402,W503,E501
 exclude = .venv,.tox
 statistics = true
 #verbose = 3
@@ -200,7 +213,7 @@ max-line-length = 88
 python =
   2.6: py26,coveralls,custom
   2.7: py27,coveralls,flake8,pylint,custom
-  3.6: py36,coveralls,black,yamllint,shellcheck,custom
+  3.6: py36,coveralls,black,yamllint,shellcheck,custom,collection
   3.7: py37,coveralls,custom
   3.8: py38,coveralls,custom
   3.8-dev: py38,coveralls,custom