This repository has been archived by the owner on Jun 6, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 549
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Alert manager] k8s cert expiration checker (#5409)
* add cert-expiration-checker cronjob * update * update * update * update * update * update * update * update * update * test * fix * update * update * update * update * update * fix * update * update * update * update * update * update * update * update * update * update * update * update * test * update * update * update * add doc * update * fix lint
- Loading branch information
Showing
11 changed files
with
176 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
10 changes: 10 additions & 0 deletions
10
src/alert-manager/build/cert-expiration-checker.common.dockerfile
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| # Copyright (c) Microsoft Corporation. | ||
| # Licensed under the MIT License. | ||
|
|
||
| FROM python:3.7 | ||
|
|
||
| COPY ./src/cert-expiration-checker . | ||
|
|
||
| RUN pip3 install -r requirements.txt | ||
|
|
||
| ENTRYPOINT ["python3", "send_alert.py"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
38 changes: 38 additions & 0 deletions
38
src/alert-manager/deploy/alert-manager-cert-expiration-check-cronjob.yaml.template
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| # Copyright (c) Microsoft Corporation. | ||
| # Licensed under the MIT License. | ||
|
|
||
| apiVersion: batch/v1beta1 | ||
| kind: CronJob | ||
| metadata: | ||
| name: cert-expiration-checker | ||
| spec: | ||
| schedule: "{{ cluster_cfg["alert-manager"]["cert-expiration-checker"]["schedule"] }}" | ||
| jobTemplate: | ||
| spec: | ||
| template: | ||
| spec: | ||
| containers: | ||
| - name: cert-expiration-checker | ||
| image: {{ cluster_cfg['cluster']['docker-registry']['prefix'] }}cert-expiration-checker:{{ cluster_cfg['cluster']['docker-registry']['tag'] }} | ||
| imagePullPolicy: Always | ||
| env: | ||
| - name: PAI_URI | ||
| {%- if "ssl" in cluster_cfg["pylon"] and cluster_cfg["pylon"]["ssl"] %} | ||
| value: "{{ cluster_cfg['pylon']['uri-https']}}" | ||
| {%- else %} | ||
| value: "{{ cluster_cfg['pylon']['uri']}}" | ||
| {%- endif %} | ||
| - name: ALERT_RESIDUAL_DAYS | ||
| value: "{{ cluster_cfg["alert-manager"]["cert-expiration-checker"]["alert-residual-days"] }}" | ||
| volumeMounts: | ||
| - mountPath: /etc/kubernetes/ssl | ||
| name: kubenetes-ssl | ||
| volumes: | ||
| - name: kubenetes-ssl | ||
| hostPath: | ||
| path: {{ cluster_cfg["alert-manager"]["cert-expiration-checker"]["cert-path"] }} | ||
| imagePullSecrets: | ||
| - name: {{ cluster_cfg["cluster"]["docker-registry"]["secret-name"] }} | ||
| restartPolicy: OnFailure | ||
| nodeSelector: | ||
| pai-master: "true" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| [SETTINGS] | ||
|
|
||
| max-line-length=140 | ||
|
|
||
| disable = | ||
| missing-docstring, | ||
| invalid-name, | ||
| cell-var-from-loop, | ||
| undefined-loop-variable, | ||
| too-many-locals, |
2 changes: 2 additions & 0 deletions
2
src/alert-manager/src/cert-expiration-checker/requirements.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| requests==2.23.0 | ||
| pyOpenSSL==20.0.1 |
65 changes: 65 additions & 0 deletions
65
src/alert-manager/src/cert-expiration-checker/send_alert.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| from datetime import timezone, datetime, timedelta | ||
| import logging | ||
| import os | ||
| import requests | ||
| import ssl | ||
| from OpenSSL import crypto | ||
|
|
||
| ALERT_PREFIX = "/alert-manager/api/v1/alerts" | ||
| APISERVER_CERT_PATH = '/etc/kubernetes/ssl/apiserver.crt' | ||
| alertResidualDays = int(os.environ.get('ALERT_RESIDUAL_DAYS')) | ||
|
|
||
| def enable_request_debug_log(func): | ||
| def wrapper(*args, **kwargs): | ||
| requests_log = logging.getLogger("urllib3") | ||
| level = requests_log.level | ||
| requests_log.setLevel(logging.DEBUG) | ||
| requests_log.propagate = True | ||
|
|
||
| try: | ||
| return func(*args, **kwargs) | ||
| finally: | ||
| requests_log.setLevel(level) | ||
| requests_log.propagate = False | ||
|
|
||
| return wrapper | ||
|
|
||
| @enable_request_debug_log | ||
| def send_alert(pai_url: str, residualTime: int, certExpirationInfo: str): | ||
| trigger_time = str(datetime.now(timezone.utc).date()) | ||
| post_url = pai_url.rstrip("/") + ALERT_PREFIX | ||
| alerts = [] | ||
| alert = { | ||
| "labels": { | ||
| "alertname": "k8s cert expiration", | ||
| "severity": "warn", | ||
| "trigger_time": trigger_time, | ||
| }, | ||
| "annotations": { | ||
| "summary": f"The k8s cert will be expired in {residualTime} days.", | ||
| "message": f"{certExpirationInfo}", | ||
| }, | ||
| "generatorURL": "alert/script", | ||
| } | ||
| alerts.append(alert) | ||
| logging.info("Sending alerts to alert-manager...") | ||
| resp = requests.post(post_url, json=alerts) | ||
| resp.raise_for_status() | ||
| logging.info("Alerts sent to alert-manager.") | ||
|
|
||
| def main(): | ||
| PAI_URI = os.environ.get("PAI_URI") | ||
| certfile = open(APISERVER_CERT_PATH).read() | ||
| cert = crypto.load_certificate(crypto.FILETYPE_PEM, certfile) | ||
| expirationTime = datetime.strptime(cert.get_notAfter().decode('ascii'), r'%Y%m%d%H%M%SZ') | ||
| delta = expirationTime - datetime.now() | ||
| if (delta < timedelta(days = alertResidualDays)): | ||
| send_alert(PAI_URI, delta.days, f'Not after {expirationTime}') | ||
|
|
||
| if __name__ == "__main__": | ||
| logging.basicConfig( | ||
| format= | ||
| "%(asctime)s - %(levelname)s - %(filename)s:%(lineno)s - %(message)s", | ||
| level=logging.INFO, | ||
| ) | ||
| main() |