From 494a6512b8027f0c31f203288d04fbe2ae3a33ad Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 22 Aug 2019 09:38:30 +0300 Subject: [PATCH 1/9] fix bug: run Copy image to ansible host cache on download_delegate host (#5094) * run 'task download_container | Copy image to ansible host cache' with synchronize on download_delegate host * try to run task copy file to ansible host on all inventory, not only on first random host --- roles/download/tasks/download_container.yml | 2 +- roles/download/tasks/download_file.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/download/tasks/download_container.yml b/roles/download/tasks/download_container.yml index cdd27fd52e0..919be714db7 100644 --- a/roles/download/tasks/download_container.yml +++ b/roles/download/tasks/download_container.yml @@ -118,10 +118,10 @@ use_ssh_args: "{{ has_bastion | default(false) }}" mode: pull delegate_facts: no - run_once: true when: - download_force_cache - not download_localhost + - download_delegate == inventory_hostname - not image_is_cached or (image_changed | default(true)) - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] diff --git a/roles/download/tasks/download_file.yml b/roles/download/tasks/download_file.yml index e2347699d2a..2aca43d5ef3 100644 --- a/roles/download/tasks/download_file.yml +++ b/roles/download/tasks/download_file.yml @@ -111,7 +111,6 @@ dest: "{{ file_path_cached }}" use_ssh_args: "{{ has_bastion | default(false) }}" mode: pull - run_once: true when: - download_force_cache - not file_is_cached or get_url_result.changed From d4f094cc11d2435561706173639e3fe02af064a7 Mon Sep 17 00:00:00 2001 From: ewtang Date: Thu, 22 Aug 2019 16:14:32 +0800 Subject: [PATCH 2/9] Add localhost to ansible.limit. (#5037) Upgrade to Kubernetes 1.15.3 (#5091) --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 88a55ed8977..a392af54435 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -206,7 +206,7 @@ Vagrant.configure("2") do |config| ansible.inventory_path = $ansible_inventory_path end ansible.become = true - ansible.limit = "all" + ansible.limit = "all,localhost" ansible.host_key_checking = false ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "-e ansible_become_pass=vagrant"] ansible.host_vars = host_vars From da089b5fca9f59029a5966351d798559abba6ef4 Mon Sep 17 00:00:00 2001 From: Victor Morales Date: Thu, 22 Aug 2019 03:16:32 -0500 Subject: [PATCH 3/9] Update CRI-O in CentOS (#4582) According to their compatibility matrix[1] the 1.11.5 version seems to be deprecated. This change updates the CentOS repository reference. [1] https://github.com/cri-o/cri-o#compatibility-matrix-cri-o---kubernetes-clusters --- README.md | 2 +- .../container-engine/cri-o/defaults/main.yml | 2 +- .../cri-o/templates/crio.conf.j2 | 327 ++++++++++-------- 3 files changed, 190 insertions(+), 141 deletions(-) diff --git a/README.md b/README.md index 1f3154af7a1..bb94f032f95 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ Supported Components - [kubernetes](https://github.com/kubernetes/kubernetes) v1.15.3 - [etcd](https://github.com/coreos/etcd) v3.3.10 - [docker](https://www.docker.com/) v18.06 (see note) - - [cri-o](http://cri-o.io/) v1.11.5 (experimental: see [CRI-O Note](docs/cri-o.md). Only on centos based OS) + - [cri-o](http://cri-o.io/) v1.14.0 (experimental: see [CRI-O Note](docs/cri-o.md). Only on centos based OS) - Network Plugin - [cni-plugins](https://github.com/containernetworking/plugins) v0.8.1 - [calico](https://github.com/projectcalico/calico) v3.7.3 diff --git a/roles/container-engine/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml index ae84e5f9984..19bbdcff409 100644 --- a/roles/container-engine/cri-o/defaults/main.yml +++ b/roles/container-engine/cri-o/defaults/main.yml @@ -1,2 +1,2 @@ --- -crio_rhel_repo_base_url: 'http://mirror.centos.org/centos/7/paas/x86_64/openshift-origin311/' +crio_rhel_repo_base_url: 'https://cbs.centos.org/repos/paas7-crio-114-candidate/x86_64/os/' diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2 index 199e348956e..303a0b0792c 100644 --- a/roles/container-engine/cri-o/templates/crio.conf.j2 +++ b/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -1,135 +1,123 @@ -# The "crio" table contains all of the server options. +# The CRI-O configuration file specifies all of the available configuration +# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime +# daemon, but in a TOML format that can be more easily modified and versioned. +# +# Please refer to crio.conf(5) for details of all configuration options. + +# CRI-O reads its storage defaults from the containers-storage.conf(5) file +# located at /etc/containers/storage.conf. Modify this storage configuration if +# you want to change the system's defaults. If you want to modify storage just +# for CRI-O, you can change the storage configuration options here. [crio] -# CRI-O reads its storage defaults from the containers/storage configuration -# file, /etc/containers/storage.conf. Modify storage.conf if you want to -# change default storage for all tools that use containers/storage. If you -# want to modify just crio, you can change the storage configuration in this -# file. - -# root is a path to the "root directory". CRIO stores all of its data, -# including container images, in this directory. +# Path to the "root directory". CRI-O stores all of its data, including +# containers images, in this directory. #root = "/var/lib/containers/storage" -# run is a path to the "run directory". CRIO stores all of its state -# in this directory. +# Path to the "run directory". CRI-O stores all of its state in this directory. #runroot = "/var/run/containers/storage" -# storage_driver select which storage driver is used to manage storage -# of images and containers. +# Storage driver used to manage the storage of images and containers. Please +# refer to containers-storage.conf(5) to see all available storage drivers. storage_driver = "overlay2" -# storage_option is used to pass an option to the storage driver. +# List to pass options to the storage driver. Please refer to +# containers-storage.conf(5) to see all available storage options. #storage_option = [ #] -# The "crio.api" table contains settings for the kubelet/gRPC interface. +# If set to false, in-memory locking will be used instead of file-based locking. +file_locking = true + +# Path to the lock file. +file_locking_path = "/run/crio.lock" + + +# The crio.api table contains settings for the kubelet/gRPC interface. [crio.api] -# listen is the path to the AF_LOCAL socket on which crio will listen. +# Path to AF_LOCAL socket on which CRI-O will listen. listen = "/var/run/crio/crio.sock" -# stream_address is the IP address on which the stream server will listen +# IP address on which the stream server will listen. stream_address = "" -# stream_port is the port on which the stream server will listen +# The port on which the stream server will listen. stream_port = "10010" -# stream_enable_tls enables encrypted tls transport of the stream server +# Enable encrypted TLS transport of the stream server. stream_enable_tls = false -# stream_tls_cert is the x509 certificate file path used to serve the encrypted stream. -# This file can change, and CRIO will automatically pick up the changes within 5 minutes. +# Path to the x509 certificate file used to serve the encrypted stream. This +# file can change, and CRI-O will automatically pick up the changes within 5 +# minutes. stream_tls_cert = "" -# stream_tls_key is the key file path used to serve the encrypted stream. -# This file can change, and CRIO will automatically pick up the changes within 5 minutes. +# Path to the key file used to serve the encrypted stream. This file can +# change, and CRI-O will automatically pick up the changes within 5 minutes. stream_tls_key = "" -# stream_tls_ca is the x509 CA(s) file used to verify and authenticate client -# communication with the tls encrypted stream. -# This file can change, and CRIO will automatically pick up the changes within 5 minutes. +# Path to the x509 CA(s) file used to verify and authenticate client +# communication with the encrypted stream. This file can change, and CRI-O will +# automatically pick up the changes within 5 minutes. stream_tls_ca = "" -# file_locking is whether file-based locking will be used instead of -# in-memory locking -file_locking = true +# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024. +grpc_max_send_msg_size = 16777216 -# The "crio.runtime" table contains settings pertaining to the OCI -# runtime used and options for how to set up and manage the OCI runtime. +# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024. +grpc_max_recv_msg_size = 16777216 + +# The crio.runtime table contains settings pertaining to the OCI runtime used +# and options for how to set up and manage the OCI runtime. [crio.runtime] -# runtime is the OCI compatible runtime used for trusted container workloads. -# This is a mandatory setting as this runtime will be the default one -# and will also be used for untrusted container workloads if -# runtime_untrusted_workload is not set. -{% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" or ansible_distribution == "Ubuntu" %} -runtime = "/usr/bin/runc" -{% else %} -runtime = "/usr/sbin/runc" -{% endif %} +# A list of ulimits to be set in containers by default, specified as +# "=:", for example: +# "nofile=1024:2048" +# If nothing is set here, settings will be inherited from the CRI-O daemon +#default_ulimits = [ +# "nofile=65536:65536", +#] -# runtime_untrusted_workload is the OCI compatible runtime used for untrusted -# container workloads. This is an optional setting, except if -# default_container_trust is set to "untrusted". -runtime_untrusted_workload = "" - -# default_workload_trust is the default level of trust crio puts in container -# workloads. It can either be "trusted" or "untrusted", and the default -# is "trusted". -# Containers can be run through different container runtimes, depending on -# the trust hints we receive from kubelet: -# - If kubelet tags a container workload as untrusted, crio will try first to -# run it through the untrusted container workload runtime. If it is not set, -# crio will use the trusted runtime. -# - If kubelet does not provide any information about the container workload trust -# level, the selected runtime will depend on the default_container_trust setting. -# If it is set to "untrusted", then all containers except for the host privileged -# ones, will be run by the runtime_untrusted_workload runtime. Host privileged -# containers are by definition trusted and will always use the trusted container -# runtime. If default_container_trust is set to "trusted", crio will use the trusted -# container runtime for all containers. -default_workload_trust = "trusted" - -# no_pivot instructs the runtime to not use pivot_root, but instead use MS_MOVE +# default_runtime is the _name_ of the OCI runtime to be used as the default. +# The name is matched against the runtimes map below. +default_runtime = "runc" + +# If true, the runtime will not use pivot_root, but instead use MS_MOVE. no_pivot = false -# conmon is the path to conmon binary, used for managing the runtime. +# Path to the conmon binary, used for monitoring the OCI runtime. conmon = "{{ crio_conmon }}" -# conmon_env is the environment variable list for conmon process, -# used for passing necessary environment variable to conmon or runtime. +# Environment variable list for the conmon process, used for passing necessary +# environment variables to conmon or the runtime. conmon_env = [ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", ] -# selinux indicates whether or not SELinux will be used for pod -# separation on the host. If you enable this flag, SELinux must be running -# on the host. +# If true, SELinux will be used for pod separation on the host. selinux = {{ (preinstall_selinux_state == 'enforcing')|lower }} -# seccomp_profile is the seccomp json profile path which is used as the -# default for the runtime. +# Path to the seccomp.json profile which is used as the default seccomp profile +# for the runtime. {% if ansible_os_family == "ClearLinux" %} seccomp_profile = "/usr/share/defaults/crio/seccomp.json" {% else %} seccomp_profile = "/etc/crio/seccomp.json" {% endif %} -# apparmor_profile is the apparmor profile name which is used as the -# default for the runtime. +# Used to change the name of the default AppArmor profile of CRI-O. The default +# profile name is "crio-default-" followed by the version string of CRI-O. apparmor_profile = "crio-default" -# cgroup_manager is the cgroup management implementation to be used -# for the runtime. +# Cgroup management implementation used for the runtime. cgroup_manager = "cgroupfs" -# default_capabilities is the list of capabilities to add and can be modified here. -# If capabilities below is commented out, the default list of capabilities defined in the -# spec will be added. -# If capabilities is empty below, only the capabilities defined in the container json -# file by the user/kube will be added. +# List of default capabilities for containers. If it is empty or commented out, +# only the capabilities defined in the containers json file by the user/kube +# will be added. default_capabilities = [ "CHOWN", "DAC_OVERRIDE", @@ -144,103 +132,164 @@ default_capabilities = [ "KILL", ] -# hooks_dir_path is the oci hooks directory for automatically executed hooks -hooks_dir_path = "/usr/share/containers/oci/hooks.d" +# List of default sysctls. If it is empty or commented out, only the sysctls +# defined in the container json file by the user/kube will be added. +default_sysctls = [ +] -# default_mounts is the mounts list to be mounted for the container when created -# deprecated, will be taken out in future versions, add default mounts to either -# /usr/share/containers/mounts.conf or /etc/containers/mounts.conf -default_mounts = [ +# List of additional devices. specified as +# "::", for example: "--device=/dev/sdc:/dev/xvdc:rwm". +#If it is empty or commented out, only the devices +# defined in the container json file by the user/kube will be added. +additional_devices = [ ] -# CRI-O reads its default mounts from the following two files: -# 1) /etc/containers/mounts.conf - this is the override file, where users can -# either add in their own default mounts, or override the default mounts shipped -# with the package. -# 2) /usr/share/containers/mounts.conf - this is the default file read for mounts. -# If you want CRI-O to read from a different, specific mounts file, you can change -# the default_mounts_file path right below. Note, if this is done, CRI-O will only add -# mounts it finds in this file. +# Path to OCI hooks directories for automatically executed hooks. +hooks_dir = [ +] -# default_mounts_file is the file path holding the default mounts to be mounted for the -# container when created. -# default_mounts_file = "" +# List of default mounts for each container. **Deprecated:** this option will +# be removed in future versions in favor of default_mounts_file. +default_mounts = [ +{% if ansible_os_family == "RedHat" %} + "/usr/share/rhel/secrets:/run/secrets", +{% endif %} +] -# pids_limit is the number of processes allowed in a container +# Path to the file specifying the defaults mounts for each container. The +# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads +# its default mounts from the following two files: +# +# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the +# override file, where users can either add in their own default mounts, or +# override the default mounts shipped with the package. +# +# 2) /usr/share/containers/mounts.conf: This is the default file read for +# mounts. If you want CRI-O to read from a different, specific mounts file, +# you can change the default_mounts_file. Note, if this is done, CRI-O will +# only add mounts it finds in this file. +# +#default_mounts_file = "" + +# Maximum number of processes allowed in a container. pids_limit = 1024 -# log_size_max is the max limit for the container log size in bytes. -# Negative values indicate that no limit is imposed. +# Maximum sized allowed for the container log file. Negative numbers indicate +# that no size limit is imposed. If it is positive, it must be >= 8192 to +# match/exceed conmon's read buffer. The file is truncated and re-opened so the +# limit is never exceeded. log_size_max = -1 -# read-only indicates whether all containers will run in read-only mode +# Whether container output should be logged to journald in addition to the kuberentes log file +log_to_journald = false + +# Path to directory in which container exit files are written to by conmon. +container_exits_dir = "/var/run/crio/exits" + +# Path to directory for container attach sockets. +container_attach_socket_dir = "/var/run/crio" + +# If set to true, all containers will run in read-only mode. read_only = false -# The "crio.image" table contains settings pertaining to the -# management of OCI images. +# Changes the verbosity of the logs based on the level it is set to. Options +# are fatal, panic, error, warn, info, and debug. +log_level = "error" -# uid_mappings specifies the UID mappings to have in the user namespace. -# A range is specified in the form containerUID:HostUID:Size. Multiple -# ranges are separed by comma. +# The UID mappings for the user namespace of each container. A range is +# specified in the form containerUID:HostUID:Size. Multiple ranges must be +# separated by comma. uid_mappings = "" -# gid_mappings specifies the GID mappings to have in the user namespace. -# A range is specified in the form containerGID:HostGID:Size. Multiple -# ranges are separed by comma. +# The GID mappings for the user namespace of each container. A range is +# specified in the form containerGID:HostGID:Size. Multiple ranges must be +# separated by comma. gid_mappings = "" +# The minimal amount of time in seconds to wait before issuing a timeout +# regarding the proper termination of the container. +ctr_stop_timeout = 0 + + # The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. + # The runtime to use is picked based on the runtime_handler provided by the CRI. + # If no runtime_handler is provided, the runtime will be picked based on the level + # of trust of the workload. + + [crio.runtime.runtimes.runc] +{% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" or ansible_distribution == "Ubuntu" %} + runtime_path = "/usr/bin/runc" +{% else %} + runtime_path = "/usr/sbin/runc" +{% endif %} + runtime_type = "oci" + + + +# The crio.image table contains settings pertaining to the management of OCI images. +# +# CRI-O reads its configured registries defaults from the system wide +# containers-registries.conf(5) located in /etc/containers/registries.conf. If +# you want to modify just CRI-O, you can change the registries configuration in +# this file. Otherwise, leave insecure_registries and registries commented out to +# use the system's defaults from /etc/containers/registries.conf. [crio.image] -# default_transport is the prefix we try prepending to an image name if the -# image name as we receive it can't be parsed as a valid source reference +# Default transport for pulling images from a remote container storage. default_transport = "docker://" -# pause_image is the image which we use to instantiate infra containers. +# The image used to instantiate infra containers. pause_image = "docker://k8s.gcr.io/pause:3.1" -# pause_command is the command to run in a pause_image to have a container just -# sit there. If the image contains the necessary information, this value need -# not be specified. +# If not empty, the path to a docker/config.json-like file containing credentials +# necessary for pulling the image specified by pause_image above. +pause_image_auth_file = "" + +# The command to run to have a container stay in the paused state. pause_command = "/pause" -# signature_policy is the name of the file which decides what sort of policy we -# use when deciding whether or not to trust an image that we've pulled. -# Outside of testing situations, it is strongly advised that this be left -# unspecified so that the default system-wide policy will be used. +# Path to the file which decides what sort of policy we use when deciding +# whether or not to trust an image that we've pulled. It is not recommended that +# this option be used, as the default behavior of using the system-wide default +# policy (i.e., /etc/containers/policy.json) is most often preferred. Please +# refer to containers-policy.json(5) for more details. {% if ansible_os_family == "ClearLinux" %} signature_policy = "/usr/share/defaults/crio/policy.json" {% else %} signature_policy = "" {% endif %} -# image_volumes controls how image volumes are handled. -# The valid values are mkdir and ignore. +# Controls how image volumes are handled. The valid values are mkdir, bind and +# ignore; the latter will ignore volumes entirely. image_volumes = "mkdir" -# CRI-O reads its configured registries defaults from the containers/image configuration -# file, /etc/containers/registries.conf. Modify registries.conf if you want to -# change default registries for all tools that use containers/image. If you -# want to modify just crio, you can change the registies configuration in this -# file. +# List of registries to be used when pulling an unqualified image (e.g., +# "alpine:latest"). By default, registries is set to "docker.io" for +# compatibility reasons. Depending on your workload and usecase you may add more +# registries (e.g., "quay.io", "registry.fedoraproject.org", +# "registry.opensuse.org", etc.). +registries = [ + "docker.io" +] +# "registry.fedoraproject.org", +# "quay.io", +# "registry.centos.org", +#] -# insecure_registries is used to skip TLS verification when pulling images. insecure_registries = [ "{{ kube_service_addresses }}" ] -# registries is used to specify a comma separated list of registries to be used -# when pulling an unqualified image (e.g. fedora:rawhide). -registries = [ - "docker.io" -] - -# The "crio.network" table contains settings pertaining to the -# management of CNI plugins. +# The crio.network table containers settings pertaining to the management of +# CNI plugins. [crio.network] -# network_dir is where CNI network configuration -# files are stored. +# Path to the directory where CNI configuration files are located. network_dir = "/etc/cni/net.d/" -# plugin_dir is where CNI plugin binaries are stored. -plugin_dir = "/opt/cni/bin/" +# Paths to directories where CNI plugin binaries are located. +plugin_dirs = [ + "/usr/libexec/cni", +{% if ansible_os_family == "ClearLinux" %} + "/opt/cni/bin/", +{% endif %} +] From 3bc4b4c1748636382ae549c9644dc99ce6725ff0 Mon Sep 17 00:00:00 2001 From: ewtang Date: Thu, 22 Aug 2019 16:46:31 +0800 Subject: [PATCH 4/9] Use raw module for bootstrap-debian.yml (#5061) Updated Openstack to terraform 0.12 (#5062) * update openstack to terraform 0.12(.5) * replace cluter.tf with cluster.tfvars * update README.md to terraform 0.12 * update Openstack CI tests to use terraform 0.12 * specify terraform version in openstack README * gitlab CI to copy cluster.tfvars in case of openstack provider * The terraform/openstack dynamic inventory can read tfstate v4 (generated by terraform 0.12) and convert them internally ro v3 (as generated by terraform 0.11.x). Additionally the script has been updated to Python 3. --- roles/bootstrap-os/tasks/bootstrap-debian.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/roles/bootstrap-os/tasks/bootstrap-debian.yml b/roles/bootstrap-os/tasks/bootstrap-debian.yml index aec6d78bdf7..74fe83a6faa 100644 --- a/roles/bootstrap-os/tasks/bootstrap-debian.yml +++ b/roles/bootstrap-os/tasks/bootstrap-debian.yml @@ -50,6 +50,20 @@ - https_proxy is defined - need_https_proxy.rc != 0 +- name: Change Network Name Resolution configuration + raw: sed -i 's/^DNSSEC=yes/DNSSEC=allow-downgrade/g' /etc/systemd/resolved.conf + become: true + environment: {} + when: + - '"bionic" in os_release.stdout' + +- name: Restart systemd-resolved service + raw: systemctl restart systemd-resolved + become: true + environment: {} + when: + - '"bionic" in os_release.stdout' + - name: Install python raw: apt-get update && \ From 07ecef86e3f81e17221d89f8ea64ce54328ebfea Mon Sep 17 00:00:00 2001 From: Michael Oglesby <44730324+mboglesby@users.noreply.github.com> Date: Thu, 22 Aug 2019 05:40:32 -0400 Subject: [PATCH 5/9] Replace fetch with synchronize due to memory error (#5084) Fix for Kubespray Issue #5059 (https://github.com/kubernetes-sigs/kubespray/issues/5059). There is a known issue with the 'fetch' module that will sometimes lead to it failing with a memory error. See ansible/ansible#11702 (https://github.com/ansible/ansible/issues/11702). I encountered this issue with the "Copy kubectl binary to ansible host" task in kubespray/roles/kubernetes/client/tasks/main.yml, and it caused my entire deployment to error out (see "Output of ansible run" above). Replacing 'fetch' with 'synchronize' fixes this issue. --- roles/kubernetes/client/tasks/main.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/roles/kubernetes/client/tasks/main.yml b/roles/kubernetes/client/tasks/main.yml index 02dc1a6faa2..0932c6d421a 100644 --- a/roles/kubernetes/client/tasks/main.yml +++ b/roles/kubernetes/client/tasks/main.yml @@ -90,11 +90,9 @@ when: kubeconfig_localhost|default(false) - name: Copy kubectl binary to ansible host - fetch: + synchronize: src: "{{ bin_dir }}/kubectl" dest: "{{ artifacts_dir }}/kubectl" - flat: yes - validate_checksum: no become: no run_once: yes when: kubectl_localhost|default(false) From f255ce3f02ecc7b507c932a96b8031e2fb2c203b Mon Sep 17 00:00:00 2001 From: Neven Miculinic Date: Thu, 22 Aug 2019 12:54:31 +0200 Subject: [PATCH 6/9] Added CRI-O support for ubuntu (#4629) * Added CRI-O support for ubuntu * implemented feedback * set crictl to fixed version * Fix errors during rebasing * Fix linting errors --- roles/container-engine/cri-o/tasks/main.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/roles/container-engine/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml index 9ab834a2b99..7eab530fb9a 100644 --- a/roles/container-engine/cri-o/tasks/main.yaml +++ b/roles/container-engine/cri-o/tasks/main.yaml @@ -30,6 +30,19 @@ state: present when: ansible_distribution in ["Ubuntu"] +- name: Add CRI-O PPA + apt_repository: + repo: ppa:projectatomic/ppa + state: present + when: ansible_distribution in ["Ubuntu"] + +- name: Install crictl + unarchive: + src: "{{ local_release_dir }}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz" + dest: "/usr/local/bin" + mode: 0755 + remote_src: yes + - name: Make sure needed folders exist in the system with_items: - /etc/crio From 7e1645845f76b8048000dd85b521d8ab56958efa Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 23 Aug 2019 10:01:19 +0300 Subject: [PATCH 7/9] Allow calico settings to be modified (#5101) Previous logic used calicoctl.sh create --skip-exists, which allowed setting initial values, but not permitting changes. --- roles/network_plugin/calico/tasks/install.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index c4b03822b79..7a920e41d8b 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -113,7 +113,7 @@ "spec": { "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}", "ipipMode": "{{ ipip_mode }}", - "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl.sh create -f - + "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl.sh apply -f - when: - inventory_hostname == groups['kube-master'][0] - 'calico_conf.stdout == "0"' @@ -131,7 +131,7 @@ "blockSize": "{{ kube_network_node_prefix }}", "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}", "ipipMode": "{{ ipip_mode }}", - "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl.sh create -f - + "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl.sh apply -f - when: - inventory_hostname == groups['kube-master'][0] - 'calico_conf.stdout == "0"' @@ -156,7 +156,7 @@ "spec": { "logSeverityScreen": "Info", "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled|default('true') }} , - "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl.sh apply -f - changed_when: false when: - inventory_hostname == groups['kube-master'][0] @@ -172,7 +172,7 @@ "spec": { "asNumber": "{{ item.as }}", "peerIP": "{{ item.router_id }}" - }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + }}' | {{ bin_dir }}/calicoctl.sh apply -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: @@ -192,7 +192,7 @@ "spec": { "nodeSelector": "!has(i-am-a-route-reflector)", "peerSelector": "has(i-am-a-route-reflector)" - }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + }}' | {{ bin_dir }}/calicoctl.sh apply -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: @@ -212,7 +212,7 @@ "spec": { "nodeSelector": "has(i-am-a-route-reflector)", "peerSelector": "has(i-am-a-route-reflector)" - }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + }}' | {{ bin_dir }}/calicoctl.sh apply -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: @@ -285,7 +285,7 @@ "asNumber": "{{ local_as }}" }, "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] - }}' | {{ bin_dir }}/calicoctl.sh {{ 'apply -f -' if calico_datastore == "kdd" else 'create --skip-exists -f -' }} + }}' | {{ bin_dir }}/calicoctl.sh apply -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" when: @@ -306,7 +306,7 @@ "asNumber": "{{ item.as }}", "node": "{{ inventory_hostname }}", "peerIP": "{{ item.router_id }}" - }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + }}' | {{ bin_dir }}/calicoctl.sh apply -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: From 10e0fe86fbf7a0f734df0ad2dc01a4e763c23cbe Mon Sep 17 00:00:00 2001 From: rptaylor Date: Fri, 23 Aug 2019 01:21:18 -0700 Subject: [PATCH 8/9] remove unimplemented custom_flags vars, document the extra_args vars (issue 4352) (#5108) --- docs/vars.md | 9 ++++++--- roles/kubernetes/master/defaults/main/main.yml | 7 ------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/vars.md b/docs/vars.md index 1bea0cabc46..7dd47912dfc 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -160,12 +160,15 @@ kubelet_custom_flags: - "--eviction-soft=memory.available<300Mi" ``` The possible vars are: -* *apiserver_custom_flags* -* *controller_mgr_custom_flags* -* *scheduler_custom_flags* * *kubelet_custom_flags* * *kubelet_node_custom_flags* +Extra flags for the API server, controller, and scheduler components can be specified using these variables, +in the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubeadm YAML config file: +* *kube_kubeadm_apiserver_extra_args* +* *kube_kubeadm_controller_extra_args* +* *kube_kubeadm_scheduler_extra_args* + #### User accounts By default, a user with admin rights is created, named `kube`. diff --git a/roles/kubernetes/master/defaults/main/main.yml b/roles/kubernetes/master/defaults/main/main.yml index 65e06c01e5c..4cdd20979af 100644 --- a/roles/kubernetes/master/defaults/main/main.yml +++ b/roles/kubernetes/master/defaults/main/main.yml @@ -136,16 +136,9 @@ kube_webhook_token_auth: false ## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication # kube_webhook_token_auth_url: https://... -## Variables for custom flags -apiserver_custom_flags: [] - # List of the preferred NodeAddressTypes to use for kubelet connections. kubelet_preferred_address_types: 'InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP' -controller_mgr_custom_flags: [] - -scheduler_custom_flags: [] - ## Extra args for k8s components passing by kubeadm kube_kubeadm_apiserver_extra_args: {} kube_kubeadm_controller_extra_args: {} From 184ac6a4e624531592e23bf452944d671cbcd8fd Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Tue, 27 Aug 2019 20:16:42 +0300 Subject: [PATCH 9/9] Parse calico nodes as json (#5114) --- roles/network_plugin/calico/rr/tasks/main.yml | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml index 0c93a55060b..5b80cf1ac41 100644 --- a/roles/network_plugin/calico/rr/tasks/main.yml +++ b/roles/network_plugin/calico/rr/tasks/main.yml @@ -3,27 +3,24 @@ include_tasks: pre.yml - name: Calico-rr | Fetch current node object - command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -oyaml" + command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -ojson" register: calico_rr_node + until: calico_rr_node is succeeded + delay: "{{ retry_stagger | random + 3 }}" + retries: 10 -# FIXME(mattymo): Use jsonpatch when ansible/ansible#52931 is merged - name: Calico-rr | Set route reflector cluster ID - shell: >- - echo -e '{{ calico_rr_node.stdout }}' | - sed '/bgp:/a \ \ \ \ routeReflectorClusterID: {{ cluster_id }}' - register: calico_rr_node - when: '("routeReflectorClusterID: " + cluster_id|string) not in calico_rr_node.stdout_lines' + set_fact: + calico_rr_node_patched: >- + {{ calico_rr_node.stdout | from_json | combine({ 'spec': { 'bgp': + { 'routeReflectorClusterID': cluster_id }}}, recursive=True) }} - name: Calico-rr | Configure route reflector - shell: |- - echo -e '{{ calico_rr_node.stdout }}' | - {{ bin_dir }}/calicoctl.sh replace -f- - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" + shell: "{{ bin_dir }}/calicoctl.sh replace -f-" + args: + stdin: "{{ calico_rr_node_patched | to_json }}" - name: Calico-rr | Set label for route reflector command: >- {{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} 'i-am-a-route-reflector=true' --overwrite - retries: 4 - delay: "{{ retry_stagger | random + 3 }}"