diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9cb4c18 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +tests/.vagrant diff --git a/README.md b/README.md index c4b6148..937f2a4 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,53 @@ # greenboot Generic Health Check Framework for systemd + +## Usage +The following directory structure is created: + +``` +/etc + /greenboot.d + /check + /required + /wanted + /green + /red +``` + +### Custom Health Checks +You have multiple options to customize greenboot’s health checking behaviour: + +* Drop scripts representing health checks that MUST NOT FAIL in order to reach a GREEN boot status into `/etc/greenboot.d/check/required`. +* Drop scripts representing health checks that MAY FAIL into `/etc/greenboot.d/check/wanted`. +* Create oneshot health check service units that MUST NOT FAIL like the following and drop them into `/etc/systemd/system` (don't forget to `systemctl enable` them afterwards): +``` +[Unit] +Description=Custom Required Health Check +Before=greenboot.target + +[Service] +Type=oneshot +ExecStart=/path/to/service + +[Install] +RequiredBy=greenboot.target +``` +* Create oneshot health check service units that MAY FAIL like the following and drop them into `/etc/systemd/system` (don't forget to `systemctl enable` them afterwards): +``` +[Unit] +Description=Custom Wanted Health Check +Before=greenboot.target + +[Service] +Type=oneshot +ExecStart=/path/to/service + +[Install] +WantedBy=greenboot.target +``` + +### Custom GREEN Status Procedures +* Drop scripts representing procedures you want to run after a GREEN boot status has been reached into `/etc/greenboot.d/green`. + +### Custom RED Status Procedures +* Drop scripts representing procedures you want to run after a RED boot status has been reached into `/etc/greenboot.d/red`. diff --git a/etc/greenboot.d/check/required/00_required_scripts_start.sh b/etc/greenboot.d/check/required/00_required_scripts_start.sh new file mode 100644 index 0000000..f02a148 --- /dev/null +++ b/etc/greenboot.d/check/required/00_required_scripts_start.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail + +echo "Running greenboot Required Scripts" diff --git a/etc/greenboot.d/check/wanted/00_wanted_scripts_start.sh b/etc/greenboot.d/check/wanted/00_wanted_scripts_start.sh new file mode 100644 index 0000000..bfee53a --- /dev/null +++ b/etc/greenboot.d/check/wanted/00_wanted_scripts_start.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail + +echo "Running greenboot Wanted Scripts" diff --git a/etc/greenboot.d/green/00_greenboot_notification.sh b/etc/greenboot.d/green/00_greenboot_notification.sh new file mode 100644 index 0000000..39f45ea --- /dev/null +++ b/etc/greenboot.d/green/00_greenboot_notification.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail + +echo "Health Check SUCCESS! Boot status is GREEN" diff --git a/etc/greenboot.d/red/00_redboot_notification.sh b/etc/greenboot.d/red/00_redboot_notification.sh new file mode 100644 index 0000000..02522b3 --- /dev/null +++ b/etc/greenboot.d/red/00_redboot_notification.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail + +echo "Health Check FAILURE! Boot status is RED" diff --git a/etc/greenboot.d/red/98_ostree_rollback.sh b/etc/greenboot.d/red/98_ostree_rollback.sh new file mode 100644 index 0000000..b267edd --- /dev/null +++ b/etc/greenboot.d/red/98_ostree_rollback.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail + +rpm-ostree rollback --reboot diff --git a/etc/greenboot.d/red/99_reboot.sh b/etc/greenboot.d/red/99_reboot.sh new file mode 100644 index 0000000..6a99691 --- /dev/null +++ b/etc/greenboot.d/red/99_reboot.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail + +reboot diff --git a/greenboot.spec b/greenboot.spec new file mode 100644 index 0000000..902592b --- /dev/null +++ b/greenboot.spec @@ -0,0 +1,111 @@ +%global github_owner LorbusChris +%global github_project greenboot +%global github_branch master +%global build_timestamp %(date +"%Y%m%d%H%M%%S") + +Name: greenboot +Version: 0.1 +Release: 1%{?dist} +Summary: Generic Health Check Framework for systemd +License: LGPLv2+ +URL: https://github.com/%{github_owner}/%{github_project} +Source0: https://github.com/%{github_owner}/%{github_project}/archive/%{github_branch}.tar.gz + +BuildArch: noarch +BuildRequires: systemd +%{?systemd_requires} +Requires: systemd + +%description +%{summary}. + +%package notifications +Summary: Notification scripts for greenboot + +%description notifications +Notification scripts for greenboot + +%package ostree +Summary: OSTree specific scripts for greenboot + +%description ostree +OSTree specific scripts for greenboot + +%package reboot +Summary: Reboot on red status for greenboot + +%description reboot +Reboot on red status for greenboot + +%prep +%setup -n %{github_project}-%{version} + +%build + +%install +install -Dpm 0755 usr/libexec/greenboot/greenboot.sh %{buildroot}%{_libexecdir}/%{name}/%{name}.sh +install -Dpm 0644 usr/lib/systemd/system/greenboot.target %{buildroot}%{_unitdir}/greenboot.target +install -Dpm 0644 usr/lib/systemd/system/greenboot-healthcheck.service %{buildroot}%{_unitdir}/greenboot-healthcheck.service +install -Dpm 0644 usr/lib/systemd/system/greenboot.service %{buildroot}%{_unitdir}/greenboot.service +install -Dpm 0644 usr/lib/systemd/system/redboot.service %{buildroot}%{_unitdir}/redboot.service +mkdir -p %{buildroot}%{_sysconfdir}/%{name}.d/check/required +install -Dpm 0755 etc/greenboot.d/check/required/00_required_scripts_start.sh %{buildroot}%{_sysconfdir}/%{name}.d/check/required/00_required_scripts_start.sh +mkdir %{buildroot}%{_sysconfdir}/%{name}.d/check/wanted +install -Dpm 0755 etc/greenboot.d/check/wanted/00_wanted_scripts_start.sh %{buildroot}%{_sysconfdir}/%{name}.d/check/wanted/00_wanted_scripts_start.sh +mkdir %{buildroot}%{_sysconfdir}/%{name}.d/green +install -Dpm 0755 etc/greenboot.d/green/00_greenboot_notification.sh %{buildroot}%{_sysconfdir}/%{name}.d/green/00_greenboot_notification.sh +mkdir %{buildroot}%{_sysconfdir}/%{name}.d/red +install -Dpm 0755 etc/greenboot.d/red/00_redboot_notification.sh %{buildroot}%{_sysconfdir}/%{name}.d/red/00_redboot_notification.sh +install -Dpm 0755 etc/greenboot.d/red/98_ostree_rollback.sh %{buildroot}%{_sysconfdir}/%{name}.d/red/98_ostree_rollback.sh +install -Dpm 0755 etc/greenboot.d/red/99_reboot.sh %{buildroot}%{_sysconfdir}/%{name}.d/red/99_reboot.sh + +%post +%systemd_post greenboot.target +%systemd_post greenboot.service +%systemd_post greenboot-healthcheck.service +%systemd_post redboot.service + +%preun +%systemd_preun greenboot.target +%systemd_preun greenboot.service +%systemd_preun greenboot-healthcheck.service +%systemd_preun redboot.service + +%postun +%systemd_postun_with_restart greenboot.target +%systemd_postun_with_restart greenboot.service +%systemd_postun_with_restart greenboot-healthcheck.service +%systemd_postun_with_restart redboot.service + +%check +# TODO + +%files +%doc README.md +%license LICENSE +%dir %{_libexecdir}/%{name} +%{_libexecdir}/%{name}/%{name}.sh +%{_unitdir}/greenboot.target +%{_unitdir}/greenboot-healthcheck.service +%{_unitdir}/greenboot.service +%{_unitdir}/redboot.service +%dir %{_sysconfdir}/%{name}.d/check/required +%dir %{_sysconfdir}/%{name}.d/check/wanted +%dir %{_sysconfdir}/%{name}.d/green +%dir %{_sysconfdir}/%{name}.d/red + +%files notifications +%{_sysconfdir}/%{name}.d/check/required/00_required_scripts_start.sh +%{_sysconfdir}/%{name}.d/check/wanted/00_wanted_scripts_start.sh +%{_sysconfdir}/%{name}.d/green/00_greenboot_notification.sh +%{_sysconfdir}/%{name}.d/red/00_redboot_notification.sh + +%files ostree +%{_sysconfdir}/%{name}.d/red/98_ostree_rollback.sh + +%files reboot +%{_sysconfdir}/%{name}.d/red/99_reboot.sh + +%changelog +* Thu Jun 14 2018 Christian Glombek - 0.1-1 +- Version 0.1 diff --git a/tests/10_failing_check.sh b/tests/10_failing_check.sh new file mode 100755 index 0000000..fc39ca2 --- /dev/null +++ b/tests/10_failing_check.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +echo "This is a failing script" + +exit 1 diff --git a/tests/Vagrantfile b/tests/Vagrantfile new file mode 100644 index 0000000..8653291 --- /dev/null +++ b/tests/Vagrantfile @@ -0,0 +1,19 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.box = "fedora/28-cloud-base" + + config.vm.provision "shell", inline: <<-SHELL + curl https://copr.fedorainfracloud.org/coprs/lorbus/greenboot/repo/fedora-28/lorbus-greenboot-fedora-28.repo --output /etc/yum.repos.d/_copr_lorbus-greenboot.repo + dnf install -y greenboot greenboot-notifications + # mv /vagrant/10_failing_check.sh /etc/greenboot.d/check/required/10_failing_check.sh + systemctl enable greenboot.target + systemctl start greenboot.target + sleep 5 + journalctl -u greenboot.target + journalctl -u greenboot + journalctl -t greenboot.sh + SHELL + +end diff --git a/tests/failing.service b/tests/failing.service new file mode 100644 index 0000000..ca0dde7 --- /dev/null +++ b/tests/failing.service @@ -0,0 +1,17 @@ +# This file is part of greenboot. +# +# greenboot is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=greenboot Failing Check Example +Before=greenboot.service + +[Service] +Type=oneshot +ExecStart=/bin/false + +[Install] +RequiredBy=greenboot.target diff --git a/usr/lib/systemd/system/greenboot-healthcheck.service b/usr/lib/systemd/system/greenboot-healthcheck.service new file mode 100644 index 0000000..0421bf3 --- /dev/null +++ b/usr/lib/systemd/system/greenboot-healthcheck.service @@ -0,0 +1,14 @@ +# This file is part of greenboot. +# +# greenboot is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=greenboot Health Check Scripts Runner +Before=greenboot.target + +[Service] +Type=oneshot +ExecStart=/usr/libexec/greenboot/greenboot.sh check diff --git a/usr/lib/systemd/system/greenboot.service b/usr/lib/systemd/system/greenboot.service new file mode 100644 index 0000000..aa76f75 --- /dev/null +++ b/usr/lib/systemd/system/greenboot.service @@ -0,0 +1,14 @@ +# This file is part of greenboot. +# +# greenboot is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=greenboot Success Scripts Runner +Requires=greenboot.target + +[Service] +Type=oneshot +ExecStart=/usr/libexec/greenboot/greenboot.sh green diff --git a/usr/lib/systemd/system/greenboot.target b/usr/lib/systemd/system/greenboot.target new file mode 100644 index 0000000..2e715da --- /dev/null +++ b/usr/lib/systemd/system/greenboot.target @@ -0,0 +1,18 @@ +# This file is part of greenboot. +# +# greenboot is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=greenboot Success Target +Wants=greenboot.service +Requires=greenboot-healthcheck.service +Before=greenboot.service +After=greenboot-healthcheck.service +OnFailure=redboot.service +OnFailureJobMode=fail + +[Install] +WantedBy=multi-user.target diff --git a/usr/lib/systemd/system/redboot.service b/usr/lib/systemd/system/redboot.service new file mode 100644 index 0000000..65cc788 --- /dev/null +++ b/usr/lib/systemd/system/redboot.service @@ -0,0 +1,14 @@ +# This file is part of greenboot. +# +# greenboot is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=greenboot Failure Scripts Runner +Conflicts=greenboot.service + +[Service] +Type=oneshot +ExecStart=/usr/libexec/greenboot/greenboot.sh red diff --git a/usr/libexec/greenboot/greenboot.sh b/usr/libexec/greenboot/greenboot.sh new file mode 100755 index 0000000..c5dcf67 --- /dev/null +++ b/usr/libexec/greenboot/greenboot.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +run_required_scripts () { + echo "Running Required Health Check Scripts..." + local required_scripts=`find /etc/greenboot.d/check/required -name '*.sh'` + local rc=0 + for script in $required_scripts; do + systemd-cat -t "$(basename $script)" bash $script || rc=$? + if [ $rc -ne 0 ]; then + echo -e "\e[1;31mRequired Health Check Script '$(basename $script)' FAILURE (exit code '$rc')\e[0m" >&2 + exit $rc + fi + echo -e "\e[1;32mRequired Health Check Script '$(basename $script)' SUCCESS\e[0m" + done +} + +run_wanted_scripts () { + echo "Running Wanted Health Check Scripts..." + local wanted_scripts=`find /etc/greenboot.d/check/wanted -name '*.sh'` + local rc=0 + for script in $wanted_scripts; do + systemd-cat -t "$(basename $script)" bash $script || rc=$? + if [ $rc -eq 0 ]; then + echo -e "\e[1;32mWanted Health Check Script '$(basename $script)' SUCCESS\e[0m" + else + echo -e "\e[1;31mWanted Health Check Script '$(basename $script)' FAILURE (exit code '$rc'). Continuing...\e[0m" >&2 + fi + done +} + +run_green_scripts () { + echo "Running Green Scripts..." + local green_scripts=`find /etc/greenboot.d/green -name '*.sh'` + local rc=0 + for script in $green_scripts; do + systemd-cat -t "$(basename $script)" bash $script || rc=$? + if [ $rc -eq 0 ]; then + echo -e "\e[1;32mGreen Script '$(basename $script)' SUCCESS\e[0m" + else + echo -e "\e[1;31mGreen Script '$(basename $script)' FAILURE (exit code '$rc'). Continuing...\e[0m" >&2 + fi + done +} + +run_red_scripts () { + echo "Running Red Scripts..." + local red_scripts=`find /etc/greenboot.d/red -name '*.sh'` + local rc=0 + for script in $red_scripts; do + systemd-cat -t "$(basename $script)" bash $script || rc=$? + if [ $rc -eq 0 ]; then + echo -e "\e[1mRed Script '$(basename $script)' SUCCESS\e[0m" + else + echo -e "\e[1;31mRed Script '$(basename $script)' FAILURE (exit code '$rc'). Continuing...\e[0m" >&2 + fi + done +} + +case "$@" in + "check") + run_required_scripts || exit 1 + run_wanted_scripts + ;; + "green") + run_green_scripts + ;; + "red") + run_red_scripts + ;; + *) + echo -e "\e[31mIllegal Command\e[0m" >&2 + exit 127 + ;; +esac