diff --git a/app/ansible.cfg b/app/ansible.cfg index e9adadd..c92960e 100644 --- a/app/ansible.cfg +++ b/app/ansible.cfg @@ -4,6 +4,7 @@ gather_subset = virtual forks=10 roles_path=/app/roles log_path=/data/ansible.log +show_custom_stats=yes # configure output callback_plugins=/app/plugins/callback @@ -11,6 +12,8 @@ stdout_callback=my_dense callback_whitelist=timer filter_plugins=/app/plugins/filter +library=/app/plugins/modules +lookup_plugins=/app/plugins/lookup [inventory] enable_plugins=script diff --git a/app/cli/deploy b/app/cli/deploy index 9adc26c..e4ad202 100755 --- a/app/cli/deploy +++ b/app/cli/deploy @@ -1,11 +1,11 @@ #!/bin/bash -export COOKBOOK=/app/cookbook.deploy -CMDS=$(cd $COOKBOOK; ls *.yml | sed 's/.yml//g' | awk '{ print " " $1; }') +export COOKBOOK=/app/deploy.d +CMDS=$(cd $COOKBOOK; ls * | awk '{ print " " $1; }') HELP=""" -USAGE: deploy RECIPE [ANSIBLE_ARGS] +USAGE: deploy COMMAND [COMMAND_ARGS] -Available Recipes: +Available Commands: help $CMDS """ @@ -32,4 +32,4 @@ export KUBECONFIG=/data/openshift-installer/auth/kubeconfig ## EXECUTE RECIPE=$1 shift -ansible-playbook ${COOKBOOK}/${RECIPE}.yml $@ +/bin/bash $COOKBOOK/$RECIPE/main.sh $@ diff --git a/app/deploy.d/container-storage/configure.py b/app/deploy.d/container-storage/configure.py new file mode 100644 index 0000000..9167755 --- /dev/null +++ b/app/deploy.d/container-storage/configure.py @@ -0,0 +1,104 @@ +import os +import sys +import pickle +import yaml +from collections import defaultdict +import ansible.parsing.yaml.objects +import ansible.utils.unsafe_proxy +from PyInquirer import Token, prompt, Separator + + +def checkbox(prompt_label, options, cache=None): + # ask for input + cached_data = [] + if cache: + try: + cached_data = pickle.load(open(cache, 'rb')) + except (IOError, EOFError): + pass + + if type(options) == list: + choices = [{'name': item, + 'checked': item in cached_data} for item in options] + else: + choices = [] + for item in options.keys(): + item = str(item) + choices += [Separator(item)] + choices += [{'name': f'{item}: {option}', + 'checked': f'{item}: {option}' in cached_data} + for option in options[item]] + questions = [ + { + 'type': 'checkbox', + 'message': str(prompt_label), + 'name': 'results', + 'choices': choices + } + ] + result = prompt(questions)['results'] + result.sort() + + # update cache + if cache and result != cached_data: + pickle.dump(result, open(cache, 'wb')) + + # nice format results + if type(options) != list: + nice_result = defaultdict(list) + for key, val in [item.split(': ') for item in result]: + nice_result[key].append(val) + return nice_result + + # return results to ansible + return result + + +def load_stats(stats_file): + return yaml.load(open(stats_file, 'r'), Loader=yaml.FullLoader) + + +def save_stats(stats_file, stats_data): + yaml.dump(stats_data, open(stats_file, 'w')) + + +def main(): + # load data + stats = load_stats(os.environ['STATS_FILE']) + + # ask for storage nodes + stg_nodes = checkbox('Select nodes to use for storage', + stats['cluster_nodes'], + '/data/container-storage-nodes.cache') + stg_nodes = [str(item) for item in stg_nodes] + + # ask for storage devices + drives = dict(zip(stg_nodes, + [stats['cluster_drives'][item] + for item in stg_nodes])) + for key in drives.keys(): + drives[key].sort() + stg_drives = checkbox('Select the block devices to use for storage', + drives, + '/data/container-storage-drives.cache') + stg_drives_out = [] + for key, val in stg_drives.items(): + stg_drives_out.append({'host': key, 'drives': val}) + + # count drives per node + drives_per_node = min([len(item['drives']) for item in stg_drives_out]) + + # save data + save_stats(os.environ['STATS_FILE'], { + 'stg_nodes': stg_nodes, + 'stg_drives': stg_drives_out, + 'cluster_nodes': stats['cluster_nodes'], + 'cluster_drives': stats['cluster_drives'], + 'drives_per_node': drives_per_node + }) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/app/deploy.d/container-storage/container-storage.yml b/app/deploy.d/container-storage/container-storage.yml new file mode 100644 index 0000000..f5cd9b6 --- /dev/null +++ b/app/deploy.d/container-storage/container-storage.yml @@ -0,0 +1,122 @@ +- name: install openshift container storage + hosts: localhost + become: no + gather_facts: no + + tasks: + - name: ensure project exists + k8s: + state: present + definition: + apiVersion: project.openshift.io/v1 + kind: Project + metadata: + name: openshift-storage + annotations: + openshift.io/cluster-monitoring: "true" + spec: + + - name: ensure operatorgroup exists + k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: openshift-storage + namespace: openshift-storage + spec: + targetNamespaces: + - openshift-storage + + - name: ensure subscription exists + k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: ocs-operator + namespace: openshift-storage + spec: + channel: "{{ lookup('ini', 'container_storage section=operators file=/app/versions.ini') }}" + name: ocs-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: "Automatic" + + - name: save clusterserviceversion object + set_fact: + k8s_obj: + apiVersion: operators.coreos.com/v1alpha1 + kind: ClusterServiceVersion + name: ocs-operator* + namepace: openshift-storage + + - name: wait for operator to become ready + assert: + that: "lookup('k8s', resource_definition=k8s_obj)[0].status.phase | default('error') == 'Succeeded'" + retries: 60 + delay: 15 + + - name: ensure storagecluster exists + k8s: + state: present + definition: + apiVersion: ocs.openshift.io/v1 + kind: StorageCluster + metadata: + name: ocs-storagecluster + namespace: openshift-storage + spec: + manageNodes: false + monDataDirHostPath: /var/lib/rook + storageDeviceSets: + - count: "{{ drives_per_node }}" + dataPVCTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: localblock + volumeMode: Block + name: ocs-deviceset + placement: {} + portable: false + replica: 3 + resources: + mds: + limits: + cpu: 0.5 + requests: + cpu: 0.25 + noobaa-core: + limits: + cpu: 0.5 + memory: 4Gi + requests: + cpu: 0.25 + memory: 2Gi + noobaa-db: + limits: + cpu: 0.5 + memory: 4Gi + requests: + cpu: 0.25 + memory: 2Gi + mon: + limits: + cpu: 0.5 + memory: 2Gi + requests: + cpu: 0.25 + memory: 1Gi + rook-ceph-rgw: + limits: + cpu: 250m + memory: 2Gi + requests: + cpu: 100m + memory: 1Gi diff --git a/app/deploy.d/container-storage/gather-facts.yml b/app/deploy.d/container-storage/gather-facts.yml new file mode 100644 index 0000000..f7edaff --- /dev/null +++ b/app/deploy.d/container-storage/gather-facts.yml @@ -0,0 +1,26 @@ +- name: query cluster facts + hosts: localhost + become: no + gather_facts: no + + pre_tasks: + - name: lookup cluster nodes + set_fact: + cluster_nodes: "{{ lookup('k8s', api_version='v1', kind='Node') | json_query('[*].metadata.name')}}" + + - name: query cluster node drives + shell: oc debug node/{{ item }} -- chroot /host lsblk -o NAME 2>/dev/null | egrep -v '((^ *`|^\|)|^NAME)' + loop: "{{ cluster_nodes }}" + register: cluster_drives + ignore_errors: yes + + - name: save discovered hosts + set_stats: + data: + cluster_nodes: "{{ cluster_nodes }}" + + - name: save discovered drives + set_stats: + data: + cluster_drives: "{{ {item.item: item.stdout_lines} }}" + loop: "{{ cluster_drives.results }}" diff --git a/app/deploy.d/container-storage/local-storage.yml b/app/deploy.d/container-storage/local-storage.yml new file mode 100644 index 0000000..686f911 --- /dev/null +++ b/app/deploy.d/container-storage/local-storage.yml @@ -0,0 +1,121 @@ +- name: install openshift local storage + hosts: localhost + become: no + gather_facts: no + + tasks: + - name: get storage node drive id paths + shell: "oc debug node/{{ item }} -- chroot /host find /dev/disk/by-id -type l -exec readlink -nf {} ';' -exec echo ': {}' ';' | egrep '(wwn|eui)' | sed 's/\\/dev\\///'" + loop: "{{ stg_drives | json_query('[*].host') }}" + register: drive_id_lkp + changed_when: no + + - name: save storage node drive id paths + set_fact: + drive_ids: "{{ drive_ids|default({}) | combine({ item.item : item.stdout | from_yaml }) }}" + loop: "{{ drive_id_lkp.results }}" + changed_when: no + + - name: ensure project exists + k8s: + state: present + definition: + apiVersion: project.openshift.io/v1 + kind: Project + metadata: + name: local-storage + spec: + + - name: ensure operatorgroup exists + k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: local-storage + namespace: local-storage + spec: + targetNamespaces: + - local-storage + + - name: ensure subscription exists + k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: local-storage-operator + namespace: local-storage + spec: + channel: "{{ lookup('ini', 'local_storage section=operators file=/app/versions.ini') }}" + name: local-storage-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: "Automatic" + + - name: save clusterserviceversion object + set_fact: + k8s_obj: + apiVersion: operators.coreos.com/v1alpha1 + kind: ClusterServiceVersion + name: local-storage-operator* + namepace: local-storage + + - name: wait for operator to become ready + assert: + that: "lookup('k8s', resource_definition=k8s_obj)[0].status.phase | default('error') == 'Succeeded'" + retries: 60 + delay: 15 + + - name: lookup storage device id paths + set_fact: + stg_devices: "{{ stg_devices|default([]) + [ drive_ids[item.0.host][item.1] ] }}" + loop: "{{ lookup('subelements', stg_drives, 'drives') }}" + + - name: ensure storage nodes are labeled + k8s: + state: present + definition: + apiVersion: v1 + kind: Node + metadata: + name: "{{ item }}" + labels: + cluster.ocs.openshift.io/openshift-storage: "" + loop: "{{ cluster_nodes }}" + when: "item in stg_nodes" + + - name: ensure non-storage nodes are not labeled + k8s: + state: present + definition: + apiVersion: v1 + kind: Node + metadata: + name: "{{ item }}" + labels: + cluster.ocs.openshift.io/openshift-storage: null + loop: "{{ cluster_nodes }}" + when: "item not in stg_nodes" + + - name: create localvolume object + k8s: + state: present + definition: + apiVersion: local.storage.openshift.io/v1 + kind: LocalVolume + metadata: + name: local-block + namespace: local-storage + spec: + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: cluster.ocs.openshift.io/openshift-storage + operator: Exists + storageClassDevices: + - storageClassName: localblock + volumeMode: Block + devicePaths: "{{ stg_devices }}" diff --git a/app/deploy.d/container-storage/main.sh b/app/deploy.d/container-storage/main.sh new file mode 100755 index 0000000..0acb5d1 --- /dev/null +++ b/app/deploy.d/container-storage/main.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +ME=$(dirname $0) + +STATS_FILE=/tmp/pipeline ansible-playbook $ME/gather-facts.yml $@ || exit 1 +STATS_FILE=/tmp/pipeline python3 $ME/configure.py $@ || exit 1 +ansible-playbook $ME/local-storage.yml -e @/tmp/pipeline $@ || exit 1 +ansible-playbook $ME/container-storage.yml -e @/tmp/pipeline $@ || exit 1 diff --git a/app/cookbook.deploy/hosted-loadbalancer.yml b/app/deploy.d/hosted-loadbalancer/hosted-loadbalancer.yml similarity index 100% rename from app/cookbook.deploy/hosted-loadbalancer.yml rename to app/deploy.d/hosted-loadbalancer/hosted-loadbalancer.yml diff --git a/app/deploy.d/hosted-loadbalancer/main.sh b/app/deploy.d/hosted-loadbalancer/main.sh new file mode 100755 index 0000000..406a75f --- /dev/null +++ b/app/deploy.d/hosted-loadbalancer/main.sh @@ -0,0 +1,3 @@ +#!/bin/bash +ME=$(dirname $0) +ansible-playbook $ME/hosted-loadbalancer.yml $@ diff --git a/app/plugins/callback/my_dense.py b/app/plugins/callback/my_dense.py index 6ca5aa2..b61072f 100644 --- a/app/plugins/callback/my_dense.py +++ b/app/plugins/callback/my_dense.py @@ -7,7 +7,10 @@ from collections import OrderedDict import sys +import os +import yaml +from ansible.utils.unsafe_proxy import AnsibleUnsafeText from ansible.module_utils.six import binary_type, text_type from ansible.module_utils.common._collections_compat import MutableMapping, MutableSequence from ansible.plugins.callback.default import CallbackModule as CallbackModule_default @@ -456,30 +459,14 @@ def v2_playbook_on_stats(self, stats): self._display.display('\n') self._display.display(line) - # In normal mode screen output should be sufficient, summary is redundant - if self._display.verbosity == 0: - return - - sys.stdout.write(vt100.bold + vt100.underline) - sys.stdout.write('SUMMARY') - - sys.stdout.write(vt100.restore + vt100.reset + '\n' + vt100.save + vt100.clearline) - sys.stdout.flush() - hosts = sorted(stats.processed.keys()) - for h in hosts: - t = stats.summarize(h) - self._display.display( - u"%s : %s %s %s %s %s %s" % ( - hostcolor(h, t), - colorize(u'ok', t['ok'], C.COLOR_OK), - colorize(u'changed', t['changed'], C.COLOR_CHANGED), - colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE), - colorize(u'failed', t['failures'], C.COLOR_ERROR), - colorize(u'rescued', t['rescued'], C.COLOR_OK), - colorize(u'ignored', t['ignored'], C.COLOR_WARN), - ), - screen_only=True - ) + # save stats + stats_file = os.environ.get('STATS_FILE') + if stats_file: + def rep_UnsafeText(dumper, data): + return dumper.represent_str(str(data)) + yaml.add_representer(AnsibleUnsafeText, rep_UnsafeText) + with open(stats_file, 'w') as fptr: + yaml.dump(stats.custom.get('_run', {}), fptr) # When using -vv or higher, simply do the default action diff --git a/app/versions.ini b/app/versions.ini new file mode 100644 index 0000000..1d33f02 --- /dev/null +++ b/app/versions.ini @@ -0,0 +1,7 @@ +[cluster] +rhcos=4.4.0 +installer=4.4.0 + +[operators] +local_storage=4.4 +container_storage=stable-4.3 diff --git a/requirements.txt b/requirements.txt index 08411a5..4cf4cdb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,6 @@ python-hpilo==4.3 # For Kubernetes cluster control openshift==0.11.0 + +# For json query +jmespath==0.10.0