Merge pull request #432 from bugy/e2e_tests

E2e tests

.gitignore

@@ -77,4 +77,5 @@ web-src/selenium-debug.log
 web-src/chromedriver.log
 web-src/geckodriver.log
 
-venv/
+venv/
+/venv2/

.travis.yml

@@ -18,30 +18,41 @@ env:
     - secure: 0AUrT+5xg+JGjAeZQfeCyVFHYw04YDSzCpeQILtt7Ca00rNJ4yshvCx+zEsyh0aMEO6+fLmGo4KCWC1tHKqKdO9ByIBBO1vbsQ83kDqC7GR1xorK8abN708NArxLdqylPRrrYK9Gr0VLk8t17DIfQdoP2QryJ2mDdsthzpliZOJ7c58LxIkBlog0uLabrX/d/m8ZEpgpqQalCUBmImc31tKBDprl5CVLk8ONLRVwdQ8WcYQaTNpOiIfx0OWp5iX+P9gMUyBTw8aFMlmwfZpXbDyGlwckDdIkKfWTUPq8FXNPLjyPbJ2zraID/kkNEw6J+x91w/F7VydhOoU/Gc4IAlL6TKN5xQquSclMz05kBHthSzZf7g6KUuQ0TgzK46zArcV4ZEItLU1h2IcLsPLi3+/O6TtUdUSQOIaCQX6YbQsnGDgEMWjtfpNJKpTyd+7SR+BuoQmtihr+Utl8rfq7vFTzkz+AiCvNnGQJzQoZKs83hgC57BRSD+LIkI0BrZ54ijNYoKCqvKHqmamkrXQEdiImBlTg8NpmBNHJgHdL6PRqi3NxLzJzdqKtz5pkI4MVVLYFXsUApD0AWvEOPejLnfVEGber5cA/Hm3HhqB7M9ja2BFml+oYBLc3mnzjKd/FT3VwWMiijfTJVvr6feFrhSQGiRyBLmaoRNmUSjUsfR0=
     # DOCKER_PASSWORD
     - secure: S22ffhnZOs1yFdBwJO9+uzy9DB3e7ehLWbj47U1zavsMKMDyDw0lkOIy8PMERFh4roBoM5dG95RIVbfNbrPXQxnY5Og2w7RTTv3eeHGdYzh+34Dppfk8mhEAhn3NL98sOe6is/5sEDvZ2ykPFLvoJmyV15V7Wvtuy1Zx+0lyZ0R0tX6sVJUWDlClHspCuSIKK+iptL5yLu4TtvX9Wks/c3kH6GIXYIJIeC63D+RRhuetbtGKND/RtFkq5IDP9qMZNXUAT/Mb8hrsk9HntFgl79dG2ChvBpDE8/LqjYDBiFTiUAtJfBhC0pVB3WaEwGTU/hWe8WTjl29JIkGcoaeT7+wncJ72lEPJoO60YWSdtWfTlNlUiN27AcxGqk39MDhB5NAbuJpKvcFLMmWFY2uJefrR6XVEXBZ+9yAwzuZmj0GYFOQTuczAqncyj/3BuOEqfIkkQ5BLAS5BUuzSEbHOjwasqbTVcWM1H3cv2ZYATXQQN8KhcZ5c5lxy8eD0NXHKvBFlS3HOXOXn3P6PqGgFHzjL+yyHMvzIXBJY4jEr8FIH16dwbXDqb4gi4lrrCZHDeIhVKsmLSUJjhmiKeP7dWcfUOGMxzLRmqA8r58TXcN3OvBrqNN63nUSG+Wb6XxmzLwE4PrlBy0fTRymG8WXrdE/Z2lglBhc8J3A8ER9c46s=
+    # AWS_ACCESS_KEY_ID
+    - secure: UUEboIaoKJD1vMGYrHzdr54QyWwOtY0XWGMrvCWwN76vHa2EOFl4OSO5MaSnNweoBfmw1HwVIeDuda1/cFVW2JPCSihVKdt9thK4dbXvXGkUOoUbHAPU9L6s2VXzFfuxTMhEew6sL8XY9K4RdVC+fgSGL03oQeJPSJgzNlAiOXEp8bNMELTEWQX7vIUaRv6vBLZ60UPkcq4SWfKb7uhLrKMGwmPVxSd7Jrp3JzXD7Tgj7WNE+KowpJqDgjuoXBhtFrqp0DCUj9HsPPSOQxuwFcQr/3u6TzpctYYv2qNor0h3ugmMJgiLNbn6VW+KbnLXGax1+YVdMGe+QAt/6yZjMbGioD1008bWeSpw2M6n7643yA9q8AeHUxMnP1VNsKit6z6YXskxsPpV9OIas/5KfmSGOhlrpeHpWonBWcdOlktVepOQ1bySxbDYH7bGx02cn+p/0P26NC+30OvejWNcUHq1n5hgT1fujxt2jGCcc0mHN+Pp4goN/9nwPExE1d55olLgFfA4Gd8gUc4bKowiBgczTYNF1Oms/8FzFWWgSwQBdPujWaEmtMkgw9bZKv8UEPeIeeyk7wZWDB61lYQ7X5X9kbE6CwTfTS4thpDlqmggRYOoHIIFKD9+QdgJcGSQCPYRnJ3Rc8XP2CbX2FcrjjDKB4VnBoK+/04Lmv1q6YM=
+    # AWS_SECRET_ACCESS_KEY
+    - secure: edY/+tpJOfRuLq6SN3pBnsKnaJux49niN+iDezEo+dsEFt0CchPQAOQ09iqIBfbF3WHrVy1jqJLwVQVbt0bZR6h2d1eGEysGF2saSEL4wMlziZfKP2aEMuVwVPgLJzt3AeoMib2tCAcXOfV1kF/lsbib3gCR0Amgb1GG3MUulAPmrwGxlwIQJuNcErMrWvl4QpTuK8/uOHBMqURCQLJLHrrYKTLzgrmbUs+7+ugqt35/hYdEZEV4lqs65Ty35eytA7zOOHR31x7k3gPx/MZyuuBHMXL8vrw1VQpDCgClp0TiT3FPJGgjf2BmkJADaAgIn+9DdxGuZrlYDorbLEvtSkGkilobL+m7WS3STqyzM7i1+MpYNMlJO+7KyXPC+RxLBsC8j4neo8SXnFZxWC83tVvMFnWXXnHb8i2wnW3E5PpB3I3ptmEsaWMb18uQa0j/G4wAKzjrxGZjWiu7goEIFKOfMPrFRj3F2uifJnTO3e/TnoKJhkAPxOkNhF6AACkhNZ9Z41Gkvav/upZxBoz7ojOoTH4Tn+KyVCMpa90wzQGvdaMuSw/+0Uf80piPO/GYriiF0pAI4yHfm9+SqtmjRtItgp7Nk3fXjYOY3tKL8jBlEUYMItMped6mYTW4SC/d2ib0xyxSNISyCZKRWKyV3rbAr6F15TVZUT+kFvBTtc0=
+    - PATH=$HOME/.local/bin:$PATH
 before_install:
-  - sudo apt-get -y install python3-pip python3-setuptools apache2-utils
+  - sudo apt-get -y install python3-pip python3-setuptools apache2-utils python3-venv
+    chromium-chromedriver
 install:
   - sudo pip3 install -r requirements.txt
   - sudo pip3 install pyasn1 --upgrade
   - sudo pip3 install ldap3 parameterized bcrypt
   - sudo pip3 install requests --upgrade
+  - sudo pip3 install awscli
   - cd web-src
   - npm install
+  - npm install allure-commandline --save-dev
   - cd ..
+  - python3 -m venv e2e_venv
+  - e2e_venv/bin/pip install -r src/e2e_tests/requirements.txt
 before_script:
   - cd src
-  - python3 -m unittest discover -s tests -p "*.py" -t .
+  #  - python3 -m unittest discover -s tests -p "*.py" -t .
   - cd ../web-src
   - npm run test:unit-ci
   - cd ..
 script:
   - python3 tools/build.py
+  - tools/run_e2e_tests.sh
+after_script:
+  - tools/report_allure.sh
 before_deploy:
-  - if ! [ "$BEFORE_DEPLOY_RUN" ]; then
-    export BEFORE_DEPLOY_RUN=1;
-
+  - |-
+    if ! [ "$BEFORE_DEPLOY_RUN" ]; then export BEFORE_DEPLOY_RUN=1;
     . tools/add_git_tag.sh
-
     fi
 deploy:
   - provider: releases
@@ -54,7 +65,7 @@ deploy:
     on:
       branch: master
   - provider: releases
-    name: $(unzip -qc build/script-server.zip version.txt)
+    name: "$(unzip -qc build/script-server.zip version.txt)"
     api_key: "$GITHUB_TOKEN"
     file: build/script-server.zip
     skip_cleanup: true
@@ -66,4 +77,4 @@ deploy:
     on:
       tags: false
       all_branches: true
-      condition: $TRAVIS_BRANCH =~ ^stable|master$
+      condition: "$TRAVIS_BRANCH =~ ^stable|master$"

samples/configs/destroy_world.json

@@ -3,5 +3,5 @@
   "script_path": "./samples/scripts/destroy_world.py",
   "description": "This is a very dangerous script, please be careful when running. Don't forget your protective helmet.",
   "requires_terminal": false,
-  "output_format": "text"
+  "output_format": "terminal"
 }

src/e2e_tests/common/pages.py

@@ -83,14 +83,26 @@ def load(self):
 
     @property
     def all_script_links(self):
-        return self.browser.find_elements_by_css_selector("a.collection-item")
+        return self.browser.find_elements_by_css_selector("a.collection-item.script-list-item")
 
     def get_script_link_by_name(self, name):
         try:
             return self.browser.find_element_by_link_text(name)
         except (NoSuchElementException, ElementNotInteractableException):
             return None
 
+    def get_scripts_group_by_name(self, name):
+        try:
+            return self.browser.find_element_by_xpath("//span[contains(text(), '{}')]/parent::a[contains(@class,'collection-item')][contains(@class,'script-group')]".format(name))
+        except (NoSuchElementException, ElementNotInteractableException):
+            return None
+
+    def get_scripts_inside_group(self, group_link):
+        try:
+            return get_parent_element(group_link).find_elements_by_css_selector("a")
+        except (NoSuchElementException, ElementNotInteractableException):
+            return None
+
     def get_random_script_link(self):
         return random.choice(self.all_script_links)
 

src/e2e_tests/conftest.py

@@ -1,20 +1,18 @@
 import json
 import time
-import pytest
 
+import allure
+import pytest
 from selenium.webdriver import Chrome, Firefox, Ie
 from selenium.webdriver.chrome.options import Options as ChromeOptions
 from selenium.webdriver.firefox.options import Options as FirefoxOptions
 
-import allure
-
-
 CONFIG_PATH = 'input/config.json'
 DEFAULT_WAIT_TIME = 10
 SUPPORTED_BROWSERS = ['chrome', 'firefox', 'ie']
 DEFAULT_HEADLESS_MODE = True
 DEFAULT_SCREENSHOTS_NEEDED = False
-DEFAULT_SCRIPTS = ["Bash formatting", "colortest", "destroy_world", "Download kittens", "Multiple words", "Very parameterized", "Write to file (WIN)"]
+DEFAULT_DISPLAYED_SCRIPTS = ["Bash formatting", "colortest", "destroy_world", "Download kittens", "Multiple words", "Very parameterized", "Write to file (WIN)"]
 
 
 with open(CONFIG_PATH) as config_file:
@@ -45,7 +43,7 @@ def config_headless_mode():
 
 @pytest.fixture(scope='session')
 def scripts():
-    return (config['scripts']) if 'scripts' in config else DEFAULT_SCRIPTS
+    return (config['scripts']) if 'scripts' in config else DEFAULT_DISPLAYED_SCRIPTS
 
 
 @pytest.fixture(scope='module')

src/e2e_tests/pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+
+log_cli = 1
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)8s %(message)s
+log_cli_date_format=%Y-%m-%d %H:%M:%S

src/e2e_tests/sample_scripts/test_destroy_world.py

@@ -1,10 +1,11 @@
-from common.pages import DestroyWorldScript
-from selenium.webdriver.common.keys import Keys
+import time
+
 import allure
+from allure import severity, severity_level
+from common.pages import DestroyWorldScript
 from common.pages import is_displayed, is_enabled, is_disabled
 from delayed_assert import expect, assert_expectations
-from allure import severity, severity_level
-import time
+from selenium.webdriver.common.keys import Keys
 
 
 @severity(severity_level.NORMAL)
@@ -99,7 +100,7 @@ def test_user_input(browser, config_host):
     destroy_world_script_page = DestroyWorldScript(browser, config_host)
 
     destroy_world_script_page.users_input.send_keys("Y" + Keys.ENTER)
-    time.sleep(1)
+    time.sleep(3)
 
     expect(destroy_world_script_page.log.get_attribute("innerHTML") == destroy_world_script_page.first_step_log_content + destroy_world_script_page.second_step_log_content)
 
@@ -117,7 +118,7 @@ def test_user_input_no(browser, config_host):
     destroy_world_script_page = DestroyWorldScript(browser, config_host)
 
     destroy_world_script_page.users_input.send_keys("N" + Keys.ENTER)
-    time.sleep(1)
+    time.sleep(3)
 
     expect(destroy_world_script_page.log.get_attribute("innerHTML") == destroy_world_script_page.first_step_log_content + destroy_world_script_page.second_step_log_content + destroy_world_script_page.no_exit_log_content)
 

src/e2e_tests/sample_scripts/test_html_group.py

@@ -0,0 +1,46 @@
+from common.pages import Page
+import allure
+from common.pages import is_displayed
+from delayed_assert import expect, assert_expectations
+from allure import severity, severity_level
+
+
+subscripts_list = ["Ploty HTML output", "Simple HTML output"]
+
+@severity(severity_level.NORMAL)
+@allure.title("Check presented scripts group 'HTML'")
+def test_presented_group_link(browser, config_host):
+    home_page = Page(browser, config_host)
+    home_page.load()
+
+    group_link = home_page.get_scripts_group_by_name("HTML")
+    expect(is_displayed(group_link), "Group 'HTML' link not displayed")
+    expect(is_displayed(home_page.find_element("i.material-icons", group_link)), "Icon 'Show more' not displayed")
+
+    assert_expectations()
+
+
+@severity(severity_level.NORMAL)
+@allure.title("Check subscripts are hide by default")
+def test_subscripts_are_hide_by_default(browser, config_host):
+    home_page = Page(browser, config_host)
+    home_page.load()
+
+    for subscript in subscripts_list:
+        expect(not is_displayed(home_page.get_script_link_by_name(subscript)), "Subscript {} is displayed by default".format(subscript))
+
+    assert_expectations()
+
+
+@severity(severity_level.NORMAL)
+@allure.title("Check subscripts are shown after click")
+def test_subscripts_are_shown_on_click(browser, config_host):
+    home_page = Page(browser, config_host)
+    home_page.load()
+
+    home_page.get_scripts_group_by_name("HTML").click()
+
+    for subscript in subscripts_list:
+        expect(is_displayed(home_page.get_script_link_by_name(subscript)), "Subscript {} is not displayed after group name click".format(subscript))
+
+    assert_expectations()

src/e2e_tests/sample_scripts/test_presented_scripts.py

@@ -15,11 +15,3 @@ def test_presented_scripts_by_name(browser, config_host, scripts):
         expect(is_displayed(home_page.get_script_link_by_name(required_script)), "Script by name \"{}\" not found".format(required_script))
 
     assert_expectations()
-
-
-@severity(severity_level.NORMAL)
-@allure.title("Check scripts amount")
-def test_presented_scripts_amount(browser, config_host, scripts):
-    home_page = Page(browser, config_host)
-
-    assert len(scripts) == len(home_page.all_script_links)

tools/build.py

@@ -15,7 +15,7 @@
 BUILD_FOLDER = 'build'
 
 
-class BuildInfo():
+class BuildInfo:
     def __init__(self):
         self.files = set()
 
@@ -101,6 +101,7 @@ def create_version_file():
 build_info.include(os.path.join('web', '**'))
 build_info.include(os.path.join('conf', 'runners'))
 build_info.exclude(os.path.join('src', 'tests'))
+build_info.exclude(os.path.join('src', 'e2e_tests'))
 build_info.exclude('tools')
 build_info.exclude('samples')
 build_info.exclude(BUILD_FOLDER)

tools/report_allure.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+aws s3 sync /tmp/allure_report "s3://script-server-tests/$TRAVIS_BRANCH/$TRAVIS_BUILD_NUMBER"
+
+allure_url="https://script-server-tests.s3-us-west-2.amazonaws.com/$TRAVIS_BRANCH/$TRAVIS_BUILD_NUMBER/index.html"
+echo "Test results: $allure_url"
+
+if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
+  curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST \
+    -d "{\"body\": \"Test results: $allure_url\"}" \
+    "https://api.github.com/repos/${TRAVIS_REPO_SLUG}/issues/${TRAVIS_PULL_REQUEST}/comments"
+fi

tools/run_e2e_tests.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+mkdir -p conf/runners
+cp samples/configs/* conf/runners/
+
+./launcher.py > /dev/null &
+SERVER_PID=$!
+
+set +e
+
+cd src/e2e_tests
+
+../../e2e_venv/bin/python -m pytest --alluredir /tmp/allure_result
+STATUS=$?
+
+kill $SERVER_PID
+rm -rf conf/runners
+
+../../web-src/node_modules/allure-commandline/dist/bin/allure generate /tmp/allure_result --clean -o /tmp/allure_report
+
+exit $STATUS

web-src/package.json

@@ -35,6 +35,7 @@
     "jest-extended": "^0.11.5",
     "jquery": "^3.4.1",
     "karma": "^4.4.1",
+    "karma-allure-reporter": "^1.4.6",
     "karma-chrome-launcher": "^2.2.0",
     "karma-firefox-launcher": "^2.1.0",
     "karma-mocha": "^1.3.0",

web-src/vue.config.js

@@ -89,7 +89,14 @@ module.exports = {
                     ]
                 }
             },
-            browsers: ['Chrome', 'Firefox']
+            browsers: ['Chrome', 'Firefox'],
+
+            reporters: ['mocha', 'allure'],
+
+            allureReport: {
+                reportDir: '/tmp/allure_result',
+                useBrowserName: true
+            }
         }
     }
 };