Feature: Pypi auto deployement (#428)

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 python:
-  - "3.5"
-  - "3.6"
+  - 3.5
+  - 3.6
 
 install:
   - pip install -r requirements.txt
@@ -13,4 +13,14 @@ script:
 
 branches:
   only:
-  - master
+    - master
+
+deploy:
+  provider: pypi
+  distributions: "sdist bdist_wheel"
+  user: aztk
+  password:
+    secure: j/mwA+hWudujDZ+JkgN3hDBUcIH3Vt1SWsvxnyUadErFsi9S9bW8OZ8kOp3R8yj73pnaENADmvDPjBt6w39b3h7l/EmNYlEdsAuPBMmaNQ0+fmPiKLCcALVIt/Odxdc+a1p2p+F3HEatqMCkaSRUs9gcHSYA1P57rt7y6i28xrCTt1ayQrdOZofgyAx4egocFQlgXgFpQIO3vNXySmc5HaIJF8h84CUKvLG/jv9bHgDoT7n2F00sQQV/tLbXdWEA2LcJFN5q4gjk3AwFtofTtMzlk3IE3y3qp9DagWbdsKoy9+b31lypVDBiIp/N+kCm/pwYDJXyG0H7U1pev2F8K/f4xF2x0AJDJJamLczAY4Ac4AMPImlI3cCpSS6htQhb+i5+KeEUW2tm+xGmdprOSto2712C4TNMna25x+WZteuPmRGYTmmDxGSeD769lE6TMGqrJTlmpkzndJiyp8ek/86hJPJUQoPfDadWdIdcA4wCjpXFBach4DYEItKvYoIk/KfK3wdqp3Lbs+MhT+JhuXGR0RRWlkCROiIo48TwQ716ddvjy2PPxwxNhplu3BdyEXLEDGN9EnEL+3L5P0GELhB4+KstjXSbECBYzULJ1+4kzvrdSB3K9F41stpT7x9PN9mK0t9FrVpXWbNozuJqkCvVVtz2ZSJ+20kvSuI1TPA=
+  on:
+    python: 3.6
+    tags: true

.vscode/settings.json

@@ -5,7 +5,8 @@
         "bin/**": true
     },
     "files.exclude": {
-        "**/__pycache__": true
+        "**/__pycache__": true,
+        "*.egg-info": true,
     },
     "python.autoComplete.extraPaths": [
         "${workspaceRoot}/node_scripts"

aztk/utils/constants.py

@@ -43,7 +43,7 @@
 """
     Source and destination paths for spark init
 """
-INIT_DIRECTORY_SOURCE = os.path.join(ROOT_PATH, 'config')
+INIT_DIRECTORY_SOURCE = os.path.join(ROOT_PATH, "aztk_cli", 'config')
 LOCAL_INIT_DIRECTORY_DEST = os.path.join(os.getcwd(), '.aztk')
 GLOBAL_INIT_DIRECTORY_DEST = os.path.join(HOME_DIRECTORY_PATH, '.aztk')
 """

aztk/version.py

@@ -22,4 +22,4 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-__version__ = '0.6.0'
+__version__ = '0.7.0b9'

aztk_cli/__init__.py

@@ -0,0 +1,2 @@
+import aztk_cli.logger
+log = aztk_cli.logger.root

cli/config.py → aztk_cli/config.py

@@ -1,6 +1,6 @@
 import os
 import yaml
-from cli import log
+from aztk_cli import log
 import aztk.spark
 from aztk.spark.models import (
     SecretsConfiguration,

cli/entrypoint.py → aztk_cli/entrypoint.py

@@ -8,8 +8,8 @@
 from typing import NamedTuple
 import azure.batch.models.batch_error as batch_error
 import aztk
-from cli import logger, log, utils, constants
-from cli.spark.endpoints import spark
+from aztk_cli import logger, log, utils, constants
+from aztk_cli.spark.endpoints import spark
 from . import plugins
 
 def main():

cli/plugins.py → aztk_cli/plugins.py

@@ -1,6 +1,6 @@
 import argparse
 import typing
-from cli import log
+from aztk_cli import log
 from aztk.models.plugins.internal import plugin_manager
 
 

cli/spark/endpoints/cluster/cluster_add_user.py → aztk_cli/spark/endpoints/cluster/cluster_add_user.py

@@ -1,8 +1,8 @@
 import argparse
 import typing
 import aztk.spark
-from cli import log
-from cli import utils, config
+from aztk_cli import log
+from aztk_cli import utils, config
 
 
 def setup_parser(parser: argparse.ArgumentParser):

cli/spark/endpoints/cluster/cluster_app_logs.py → aztk_cli/spark/endpoints/cluster/cluster_app_logs.py

@@ -2,7 +2,7 @@
 import typing
 import aztk
 
-from cli import utils, config
+from aztk_cli import utils, config
 
 def setup_parser(parser: argparse.ArgumentParser):
     parser.add_argument('--id',

cli/spark/endpoints/cluster/cluster_copy.py → aztk_cli/spark/endpoints/cluster/cluster_copy.py

@@ -1,7 +1,7 @@
 import argparse
 import typing
 import aztk.spark
-from cli import config
+from aztk_cli import config
 
 
 def setup_parser(parser: argparse.ArgumentParser):

cli/spark/endpoints/cluster/cluster_create.py → aztk_cli/spark/endpoints/cluster/cluster_create.py

@@ -3,9 +3,9 @@
 import typing
 import aztk.spark
 from aztk.spark.models import ClusterConfiguration, UserConfiguration
-from cli import log
-from cli.config import load_aztk_spark_config
-from cli import utils, config
+from aztk_cli import log
+from aztk_cli.config import load_aztk_spark_config
+from aztk_cli import utils, config
 
 
 def setup_parser(parser: argparse.ArgumentParser):

cli/spark/endpoints/cluster/cluster_delete.py → aztk_cli/spark/endpoints/cluster/cluster_delete.py

@@ -1,7 +1,7 @@
 import argparse
 import typing
 import aztk
-from cli import log, config
+from aztk_cli import log, config
 
 
 def setup_parser(parser: argparse.ArgumentParser):

cli/spark/endpoints/cluster/cluster_get.py → aztk_cli/spark/endpoints/cluster/cluster_get.py

@@ -1,8 +1,8 @@
 import argparse
 import typing
 import aztk
-from cli import log
-from cli import utils, config
+from aztk_cli import log
+from aztk_cli import utils, config
 
 
 def setup_parser(parser: argparse.ArgumentParser):

cli/spark/endpoints/cluster/cluster_list.py → aztk_cli/spark/endpoints/cluster/cluster_list.py

@@ -1,7 +1,7 @@
 import argparse
 import typing
 import aztk
-from cli import utils, config
+from aztk_cli import utils, config
 
 
 def setup_parser(_: argparse.ArgumentParser):

cli/spark/endpoints/cluster/cluster_run.py → aztk_cli/spark/endpoints/cluster/cluster_run.py

@@ -1,7 +1,7 @@
 import argparse
 import typing
 import aztk.spark
-from cli import utils, config
+from aztk_cli import utils, config
 
 def setup_parser(parser: argparse.ArgumentParser):
     parser.add_argument('--id',

cli/spark/endpoints/cluster/cluster_ssh.py → aztk_cli/spark/endpoints/cluster/cluster_ssh.py

@@ -1,8 +1,8 @@
 import argparse
 import typing
-from cli import log
-from cli import utils, config
-from cli.config import SshConfig
+from aztk_cli import log
+from aztk_cli import utils, config
+from aztk_cli.config import SshConfig
 import aztk
 import azure.batch.models.batch_error as batch_error
 from aztk.models import ClusterConfiguration

cli/spark/endpoints/cluster/cluster_submit.py → aztk_cli/spark/endpoints/cluster/cluster_submit.py

@@ -1,6 +1,6 @@
 import argparse
 import typing
-from cli import utils, config, log
+from aztk_cli import utils, config, log
 import aztk.spark
 
 

cli/spark/endpoints/job/delete.py → aztk_cli/spark/endpoints/job/delete.py

@@ -1,7 +1,7 @@
 import argparse
 import typing
 import aztk.spark
-from cli import log, config
+from aztk_cli import log, config
 
 
 def setup_parser(parser: argparse.ArgumentParser):

cli/spark/endpoints/job/get.py → aztk_cli/spark/endpoints/job/get.py

@@ -2,8 +2,8 @@
 import typing
 import time
 import aztk.spark
-from cli import config
-from cli import utils
+from aztk_cli import config
+from aztk_cli import utils
 
 def setup_parser(parser: argparse.ArgumentParser):
     parser.add_argument('--id',

cli/spark/endpoints/job/get_app.py → aztk_cli/spark/endpoints/job/get_app.py

@@ -2,8 +2,8 @@
 import typing
 import time
 import aztk.spark
-from cli import config
-from cli import utils
+from aztk_cli import config
+from aztk_cli import utils
 
 def setup_parser(parser: argparse.ArgumentParser):
     parser.add_argument('--id',

cli/spark/endpoints/job/get_app_logs.py → aztk_cli/spark/endpoints/job/get_app_logs.py

@@ -1,7 +1,7 @@
 import argparse
 import typing
 import aztk.spark
-from cli import utils, config
+from aztk_cli import utils, config
 
 def setup_parser(parser: argparse.ArgumentParser):
     parser.add_argument('--id',

cli/spark/endpoints/job/list.py → aztk_cli/spark/endpoints/job/list.py

@@ -2,8 +2,8 @@
 import typing
 import time
 import aztk.spark
-from cli import config
-from cli import utils
+from aztk_cli import config
+from aztk_cli import utils
 
 def setup_parser(_: argparse.ArgumentParser):
     # No arguments for list yet

cli/spark/endpoints/job/list_apps.py → aztk_cli/spark/endpoints/job/list_apps.py

@@ -1,8 +1,8 @@
 import argparse
 import typing
 import aztk.spark
-from cli import config
-from cli import utils
+from aztk_cli import config
+from aztk_cli import utils
 
 def setup_parser(parser: argparse.ArgumentParser):
     parser.add_argument('--id',

cli/spark/endpoints/job/stop.py → aztk_cli/spark/endpoints/job/stop.py

@@ -2,8 +2,8 @@
 import typing
 import time
 import aztk.spark
-from cli import config
-from cli import utils
+from aztk_cli import config
+from aztk_cli import utils
 
 def setup_parser(parser: argparse.ArgumentParser):
     parser.add_argument('--id',

cli/spark/endpoints/job/stop_app.py → aztk_cli/spark/endpoints/job/stop_app.py

@@ -2,9 +2,9 @@
 import typing
 import time
 import aztk.spark
-from cli import config
-from cli import utils
-from cli import log
+from aztk_cli import config
+from aztk_cli import utils
+from aztk_cli import log
 
 
 def setup_parser(parser: argparse.ArgumentParser):

cli/spark/endpoints/job/submit.py → aztk_cli/spark/endpoints/job/submit.py

@@ -2,8 +2,8 @@
 import typing
 import time
 import aztk.spark
-from cli import config, utils, log
-from cli.config import JobConfig, load_aztk_spark_config
+from aztk_cli import config, utils, log
+from aztk_cli.config import JobConfig, load_aztk_spark_config
 
 
 def setup_parser(parser: argparse.ArgumentParser):

requirements.txt

@@ -12,3 +12,4 @@ yapf==0.20.1
 pylint==1.8.2
 pytest==3.1.3
 pytest-xdist==1.22.0
+twine==1.9.1

setup.py

@@ -1,22 +1,59 @@
+import os
 from setuptools import setup, find_packages
-from cli import constants
+from aztk_cli import constants
 from aztk import version
 
+data_files = []
 
-setup(name='aztk',
-      version=version.__version__,
-      description='Utility for data engineers or platform developers to Run distributed jobs in Azure',
-      url='<tbd>',
-      author='Microsoft',
-      author_email='[email protected]',
-      license='MIT',
-      packages=find_packages(),
-      scripts=[
-      #     'bin/aztk',
-      ],
-      entry_points=dict(
-            console_scripts=[
-                  "{0} = cli.entrypoint:main".format(constants.CLI_EXE)
-            ]
-      ),
-      zip_safe=False)
+
+def find_package_files(root, directory, dest=""):
+    paths = []
+    for (path, _, filenames) in os.walk(os.path.join(root, directory)):
+        for filename in filenames:
+            paths.append(os.path.relpath(os.path.join(dest, path, filename), root))
+    return paths
+
+
+with open('README.md') as fd:
+    long_description = fd.read()
+
+setup(
+    name='aztk',
+    version=version.__version__,
+    description='On-demand, Dockerized, Spark Jobs on Azure (powered by Azure Batch)',
+    long_description=long_description,
+    url='https://github.com/Azure/aztk',
+    author='Microsoft',
+    author_email='[email protected]',
+    license='MIT',
+    packages=find_packages(exclude=["tests"]),
+    install_requires=[
+        "azure-batch==3.0.0",
+        "azure-mgmt-batch==5.0.0",
+        "azure-mgmt-storage==1.5.0",
+        "azure-storage==0.33.0",
+        "pyyaml>=3.12",
+        "pycryptodome>=3.4",
+        "paramiko>=2.4",
+    ],
+    package_data={
+        'aztk': find_package_files("", "node_scripts", ".."),
+        'aztk_cli': find_package_files("aztk_cli", "config"),
+    },
+    scripts=[],
+    entry_points=dict(console_scripts=["{0} = aztk_cli.entrypoint:main".format(constants.CLI_EXE)]),
+    zip_safe=False,
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+    ],
+    project_urls={
+        'Documentation': 'https://github.com/Azure/aztk/wiki/',
+        'Source': 'https://github.com/Azure/aztk/',
+        'Tracker': 'https://github.com/Azure/aztk/issues',
+    },
+    python_requires='>=3.5',
+)

tests/spark/integration_tests/sdk/cluster/test_cluster.py

@@ -9,7 +9,7 @@
 import pytest
 from aztk.utils import constants
 from aztk.error import AztkError
-from cli import config
+from aztk_cli import config
 
 
 # base cluster name

tests/spark/integration_tests/sdk/job/test_job.py

@@ -5,7 +5,7 @@
 
 import aztk.spark
 from aztk.error import AztkError
-from cli import config
+from aztk_cli import config
 
 dt = datetime.now()
 time = dt.microsecond