Skip to content

Commit

Permalink
Merge pull request #54326 from Akm0d/neon_cloud_mf
Browse files Browse the repository at this point in the history 
Merged cloud test refactor from 2019.2.1
  • Loading branch information
@dwoz
dwoz authored Aug 30, 2019
2 parents 1ded865 + 52739cc commit f818ea6
Show file tree
Hide file tree
Showing 36 changed files with 914 additions and 2,399 deletions.
214 changes: 96 additions & 118 deletions salt/cloud/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,6 @@ def _call(queue, args, kwargs):
queue.put('ERROR')
queue.put('Exception')
queue.put('{0}\n{1}\n'.format(ex, trace))
except SystemExit as ex:
trace = traceback.format_exc()
queue.put('ERROR')
queue.put('System exit')
queue.put('{0}\n{1}\n'.format(ex, trace))
return ret
return _call

Expand Down Expand Up @@ -189,10 +184,8 @@ class CloudClient(object):
def __init__(self, path=None, opts=None, config_dir=None, pillars=None):
if opts:
self.opts = opts
elif path:
self.opts = salt.config.cloud_config(path)
else:
self.opts = salt.config.cloud_config()
self.opts = salt.config.cloud_config(path)

# Check the cache-dir exists. If not, create it.
v_dirs = [self.opts['cachedir']]
Expand Down Expand Up @@ -524,7 +517,6 @@ class Cloud(object):
'''
def __init__(self, opts):
self.opts = opts
self.client = CloudClient(opts=self.opts)
self.clouds = salt.loader.clouds(self.opts)
self.__filter_non_working_providers()
self.__cached_provider_queries = {}
Expand Down Expand Up @@ -2058,138 +2050,124 @@ def run_map(self, dmap):

# Now sort the create list based on dependencies
create_list = sorted(six.iteritems(dmap['create']), key=lambda x: x[1]['level'])
full_map = dmap['create'].copy()
if 'existing' in dmap:
full_map.update(dmap['existing'])
possible_master_list = sorted(six.iteritems(full_map), key=lambda x: x[1]['level'])
output = {}
if self.opts['parallel']:
parallel_data = []
master_name = None
master_minion_name = None
master_host = None
master_finger = None
for name, profile in possible_master_list:
if profile.get('make_master', False) is True:
master_name = name
master_profile = profile

if master_name:
# If the master already exists, get the host
if master_name not in dmap['create']:
master_host = self.client.query()
for provider_part in master_profile['provider'].split(':'):
master_host = master_host[provider_part]
master_host = master_host[master_name][master_profile.get('ssh_interface', 'public_ips')]
if not master_host:
raise SaltCloudSystemExit(
'Could not get the hostname of master {}.'.format(master_name)
)
# Otherwise, deploy it as a new master
else:
master_minion_name = master_name
log.debug('Creating new master \'%s\'', master_name)
if salt.config.get_cloud_config_value(
'deploy',
try:
master_name, master_profile = next((
(name, profile) for name, profile in create_list
if profile.get('make_master', False) is True
))
master_minion_name = master_name
log.debug('Creating new master \'%s\'', master_name)
if salt.config.get_cloud_config_value(
'deploy',
master_profile,
self.opts
) is False:
raise SaltCloudSystemExit(
'Cannot proceed with \'make_master\' when salt deployment '
'is disabled(ex: --no-deploy).'
)

# Generate the master keys
log.debug('Generating master keys for \'%s\'', master_profile['name'])
priv, pub = salt.utils.cloud.gen_keys(
salt.config.get_cloud_config_value(
'keysize',
master_profile,
self.opts
) is False:
raise SaltCloudSystemExit(
'Cannot proceed with \'make_master\' when salt deployment '
'is disabled(ex: --no-deploy).'
)
)
)
master_profile['master_pub'] = pub
master_profile['master_pem'] = priv

# Generate the master keys
log.debug('Generating master keys for \'%s\'', master_profile['name'])
# Generate the fingerprint of the master pubkey in order to
# mitigate man-in-the-middle attacks
master_temp_pub = salt.utils.files.mkstemp()
with salt.utils.files.fopen(master_temp_pub, 'w') as mtp:
mtp.write(pub)
master_finger = salt.utils.crypt.pem_finger(master_temp_pub, sum_type=self.opts['hash_type'])
os.unlink(master_temp_pub)

if master_profile.get('make_minion', True) is True:
master_profile.setdefault('minion', {})
if 'id' in master_profile['minion']:
master_minion_name = master_profile['minion']['id']
# Set this minion's master as local if the user has not set it
if 'master' not in master_profile['minion']:
master_profile['minion']['master'] = '127.0.0.1'
if master_finger is not None:
master_profile['master_finger'] = master_finger

# Generate the minion keys to pre-seed the master:
for name, profile in create_list:
make_minion = salt.config.get_cloud_config_value(
'make_minion', profile, self.opts, default=True
)
if make_minion is False:
continue

log.debug('Generating minion keys for \'%s\'', profile['name'])
priv, pub = salt.utils.cloud.gen_keys(
salt.config.get_cloud_config_value(
'keysize',
master_profile,
profile,
self.opts
)
)
master_profile['master_pub'] = pub
master_profile['master_pem'] = priv

# Generate the fingerprint of the master pubkey in order to
# mitigate man-in-the-middle attacks
master_temp_pub = salt.utils.files.mkstemp()
with salt.utils.files.fopen(master_temp_pub, 'w') as mtp:
mtp.write(pub)
master_finger = salt.utils.crypt.pem_finger(master_temp_pub, sum_type=self.opts['hash_type'])
os.unlink(master_temp_pub)

if master_profile.get('make_minion', True) is True:
master_profile.setdefault('minion', {})
if 'id' in master_profile['minion']:
master_minion_name = master_profile['minion']['id']
# Set this minion's master as local if the user has not set it
if 'master' not in master_profile['minion']:
master_profile['minion']['master'] = '127.0.0.1'
if master_finger is not None:
master_profile['master_finger'] = master_finger

# Generate the minion keys to pre-seed the master:
for name, profile in create_list:
make_minion = salt.config.get_cloud_config_value(
'make_minion', profile, self.opts, default=True
)
if make_minion is False:
continue
profile['pub_key'] = pub
profile['priv_key'] = priv
# Store the minion's public key in order to be pre-seeded in
# the master
master_profile.setdefault('preseed_minion_keys', {})
master_profile['preseed_minion_keys'].update({name: pub})

local_master = False
if master_profile['minion'].get('local_master', False) and \
master_profile['minion'].get('master', None) is not None:
# The minion is explicitly defining a master and it's
# explicitly saying it's the local one
local_master = True

log.debug('Generating minion keys for \'%s\'', profile['name'])
priv, pub = salt.utils.cloud.gen_keys(
salt.config.get_cloud_config_value(
'keysize',
profile,
self.opts
)
)
profile['pub_key'] = pub
profile['priv_key'] = priv
# Store the minion's public key in order to be pre-seeded in
# the master
master_profile.setdefault('preseed_minion_keys', {})
master_profile['preseed_minion_keys'].update({name: pub})

local_master = False
if master_profile['minion'].get('local_master', False) and \
master_profile['minion'].get('master', None) is not None:
# The minion is explicitly defining a master and it's
# explicitly saying it's the local one
local_master = True

out = self.create(master_profile, local_master=local_master)

if not isinstance(out, dict):
log.debug('Master creation details is not a dictionary: %s', out)

elif 'Errors' in out:
raise SaltCloudSystemExit(
'An error occurred while creating the master, not '
'continuing: {0}'.format(out['Errors'])
out = self.create(master_profile, local_master=local_master)

if not isinstance(out, dict):
log.debug(
'Master creation details is not a dictionary: {0}'.format(
out
)
)

deploy_kwargs = (
self.opts.get('show_deploy_args', False) is True and
# Get the needed data
out.get('deploy_kwargs', {}) or
# Strip the deploy_kwargs from the returned data since we don't
# want it shown in the console.
out.pop('deploy_kwargs', {})
elif 'Errors' in out:
raise SaltCloudSystemExit(
'An error occurred while creating the master, not '
'continuing: {0}'.format(out['Errors'])
)

master_host = deploy_kwargs.get('salt_host', deploy_kwargs.get('host', None))
if master_host is None:
raise SaltCloudSystemExit(
'Host for new master {0} was not found, '
'aborting map'.format(
master_name
)
deploy_kwargs = (
self.opts.get('show_deploy_args', False) is True and
# Get the needed data
out.get('deploy_kwargs', {}) or
# Strip the deploy_kwargs from the returned data since we don't
# want it shown in the console.
out.pop('deploy_kwargs', {})
)

master_host = deploy_kwargs.get('salt_host', deploy_kwargs.get('host', None))
if master_host is None:
raise SaltCloudSystemExit(
'Host for new master {0} was not found, '
'aborting map'.format(
master_name
)
output[master_name] = out
else:
)
output[master_name] = out
except StopIteration:
log.debug('No make_master found in map')
# Local master?
# Generate the fingerprint of the master pubkey in order to
Expand Down
8 changes: 3 additions & 5 deletions salt/cloud/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import salt.utils.parsers
import salt.utils.user
from salt.exceptions import SaltCloudException, SaltCloudSystemExit
from salt.utils.verify import check_user, verify_env, verify_files, verify_log
from salt.utils.verify import check_user, verify_env, verify_log_files, verify_log

# Import 3rd-party libs
from salt.ext import six
Expand Down Expand Up @@ -70,11 +70,9 @@ def run(self):
root_dir=self.config['root_dir'],
)
logfile = self.config['log_file']
if logfile is not None and not logfile.startswith('tcp://') \
and not logfile.startswith('udp://') \
and not logfile.startswith('file://'):
if logfile is not None:
# Logfile is not using Syslog, verify
verify_files([logfile], salt_master_user)
verify_log_files([logfile], salt_master_user)
except (IOError, OSError) as err:
log.error('Error while verifying the environment: %s', err)
sys.exit(err.errno)
Expand Down
48 changes: 18 additions & 30 deletions salt/cloud/clouds/azurearm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,6 @@
* ``client_id``
* ``secret``
if using MSI-style authentication:
* ``subscription_id``
Optional provider parameters:
**cloud_environment**: Used to point the cloud driver to different API endpoints, such as Azure GovCloud. Possible values:
Expand Down Expand Up @@ -254,22 +251,14 @@ def __is_provider_configured(opts, provider, required_keys=()):
provider = __is_provider_configured(
__opts__,
__active_provider_name__ or __virtualname__,
('subscription_id', 'tenant', 'client_id', 'secret'),
('subscription_id', 'tenant', 'client_id', 'secret')
)

if provider is False:
provider = __is_provider_configured(
__opts__,
__active_provider_name__ or __virtualname__,
('subscription_id', 'username', 'password'),
)

if provider is False:
# check if using MSI style credentials...
provider = config.is_provider_configured(
__opts__,
__active_provider_name__ or __virtualname__,
required_keys=('subscription_id',),
('subscription_id', 'username', 'password')
)

return provider
Expand Down Expand Up @@ -322,13 +311,11 @@ def get_conn(client_type):
)
conn_kwargs.update({'client_id': client_id, 'secret': secret,
'tenant': tenant})

username = config.get_cloud_config_value(
'username',
get_configured_provider(), __opts__, search_global=False
)

if username is not None:
else:
username = config.get_cloud_config_value(
'username',
get_configured_provider(), __opts__, search_global=False
)
password = config.get_cloud_config_value(
'password',
get_configured_provider(), __opts__, search_global=False
Expand Down Expand Up @@ -1048,15 +1035,6 @@ def request_instance(vm_):
default=False
)

vm_password = salt.utils.stringutils.to_str(
config.get_cloud_config_value(
'ssh_password', vm_, __opts__, search_global=True,
default=config.get_cloud_config_value(
'win_password', vm_, __opts__, search_global=True
)
)
)

os_kwargs = {}
win_installer = config.get_cloud_config_value(
'win_installer', vm_, __opts__, search_global=True
Expand All @@ -1074,6 +1052,16 @@ def request_instance(vm_):
ssh=sshconfiguration,
)
os_kwargs['linux_configuration'] = linuxconfiguration
vm_password = None
else:
vm_password = salt.utils.stringutils.to_str(
config.get_cloud_config_value(
'ssh_password', vm_, __opts__, search_global=True,
default=config.get_cloud_config_value(
'win_password', vm_, __opts__, search_global=True
)
)
)

if win_installer or (vm_password is not None and not disable_password_authentication):
if not isinstance(vm_password, str):
Expand Down Expand Up @@ -1340,7 +1328,7 @@ def request_instance(vm_):
),
network_profile=NetworkProfile(
network_interfaces=[
NetworkInterfaceReference(vm_['iface_id']),
NetworkInterfaceReference(id=vm_['iface_id']),
],
),
availability_set=availability_set,
Expand Down
Loading

0 comments on commit f818ea6

Please sign in to comment.