Add permission and new API in log-manager (#5046)

Add permission check in log-manage
Add log-manager API to retrieve log

contrib/kubespray/quick-start/services-configuration.yaml.template

@@ -238,6 +238,10 @@ authentication:
 # uncomment following section if you want to customize the port of log-manager
 # log-manager:
 #  port: 9103
+#  admin_name: "admin"
+#  admin_password: "admin"
+#  jwt_secret: "jwt_secret"
+#  token_expired_second: 120
 
 
 # uncomment following section if you want to customize the port of storage-manager

deployment/quick-start/services-configuration.yaml.template

@@ -128,6 +128,10 @@ rest-server:
 # uncomment following section if you want to customize the port of log-manager
 # log-manager:
 #  port: 9103
+#  admin_name: "admin"
+#  admin_password: "admin"
+#  jwt_secret: "jwt_secret"
+#  token_expired_second: 120
 
 # uncomment following section if you want to customize the port of storage-manager
 # storage-manager:

src/log-manager/build/log-manager-cleaner.k8s.dockerfile

@@ -0,0 +1,24 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+FROM alpine:3.10
+
+# install dev tools
+RUN apk update && apk add --no-cache tini bash findutils
+COPY src/cleaner/ /usr/bin/cleaner/
+ENTRYPOINT ["/sbin/tini","--","/usr/bin/cleaner/entrypoint.sh"]
+

src/log-manager/build/log-manager-nginx.k8s.dockerfile

@@ -15,5 +15,11 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-FROM openresty/openresty:1.15.8.2-alpine
-COPY src/nginx/nginx.conf /etc/nginx/conf.d/default.conf
+FROM openresty/openresty:1.15.8.3-2-alpine-fat
+
+RUN luarocks install lua-cjson && luarocks install lua-resty-jwt && \
+  luarocks install luafilesystem
+
+COPY src/nginx/nginx.conf.default /etc/nginx/conf.d/default.conf
+COPY src/nginx/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf
+COPY src/nginx/*.lua /etc/nginx/lua/

src/log-manager/config/log-manager.md

@@ -1,46 +1,43 @@
 ## Log-manager section parser
 
-- [Default Configuration](#D_Config)
-- [How to Configure](#HT_Config)
-- [Generated Configuration](#G_Config)
-- [Data Table](#T_config)
+- [Default configuration](#default-configuration)
+- [How to configure cluster section in service-configuration.yaml](#how-to-configure-cluster-section-in-service-configurationyaml)
+- [Generated Configuration](#generated-configuration)
+- [Table](#table)
 
-#### Default configuration <a name="D_Config"></a>
+#### Default configuration
 
 [log-manager default configuration](log-manager.yaml)
 
-#### How to configure cluster section in service-configuration.yaml <a name="HT_Config"></a>
+#### How to configure cluster section in service-configuration.yaml
 
 All configurations in this section is optional. If you want to customized these value, you can configure it in service-configuration.yaml.
 
 For example, if you want to use different port than the default 9103, add following to your service-configuration.yaml as following:
 ```yaml
 log-manager:
-    port: new-value
+  port: new-value
 ```
 
-#### Generated Configuration <a name="G_Config"></a>
+#### Generated Configuration
 
 Generated configuration means the object model after parsing. The parsed data will be presented by a yaml format.
 ```yaml
 log-manager:
-    port: 9103
+  port: 9103
+  admin_name: admin
+  admin_password: admin
+  jwt_secret: "jwt_secret"
+  token_expired_second: 120
 ```
 
 
-#### Table <a name="T_Config"></a>
-
-<table>
-<tr>
-    <td>Data in Configuration File</td>
-    <td>Data in Cluster Object Model</td>
-    <td>Data in Jinja2 Template</td>
-    <td>Data type</td>
-</tr>
-<tr>
-    <td>log-manager.port</td>
-    <td>com["log-manager"]["port"]</td>
-    <td>cluster_cfg["log-manager"]["port"]</td>
-    <td>Int</td>
-</tr>
-</table>
+#### Table
+
+| Data in Configuration File        | Data in Cluster Object Model                | Data in Jinja2 Template                            | Data type |
+|-----------------------------------|---------------------------------------------|----------------------------------------------------|-----------|
+| log-manager.port                  | com["log-manager"]["port"]                  | cluster_cfg["log-manager"]["port"]                 | Int       |
+| log-manager.admin_name            | com["log-manager"]["admin_name"]            | cluster_cfg["log-manager"]["admin_name"]           | String    |
+| log-manager.admin_password        | com["log-manager"]["admin_password"]        | cluster_cfg["log-manager"]["admin_password"]       | String    |
+| log-manager.jwt_secret            | com["log-manager"]["jwt_secret"]            | cluster_cfg["log-manager"]["jwt_secret"]           | String    |
+| log-manager.token_expired_second  | com["log-manager"]["token_expired_second"]  | cluster_cfg["log-manager"]["token_expired_second"] | Int       |

src/log-manager/config/log-manager.yaml

@@ -18,3 +18,7 @@
 service_type: "k8s"
 
 port: 9103
+admin_name: "admin"
+admin_password: "admin"
+jwt_secret: "jwt_secret"
+token_expired_second: 120

src/log-manager/deploy/log-manager.yaml.template

@@ -34,12 +34,9 @@ spec:
       priorityClassName: pai-daemon-priority
       hostNetwork: false
       containers:
-      - name: log-manager-logrotate
-        image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}log-manager-logrotate:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
+      - name: log-cleaner
+        image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}log-manager-cleaner:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
         imagePullPolicy: Always
-        env:
-        - name: LOGROTATE_CRONSCHEDULE
-          value: "*/10 * * * *"
         volumeMounts:
         - name: pai-log
           mountPath: /usr/local/pai/logs
@@ -77,6 +74,19 @@ spec:
             cpu: 0
             memory: "128Mi"
         {%- endif %}
+        env:
+        - name: ADMIN_NAME
+          value: {{ cluster_cfg["log-manager"]["admin_name"] }}
+        - name: ADMIN_PASSWORD
+          value: {{ cluster_cfg["log-manager"]["admin_password"] }}
+        - name: JWT_SECRET
+          value: {{ cluster_cfg["log-manager"]["jwt_secret"] }}
+        - name: TOKEN_EXPIRED_SECOND
+          value: '{{ cluster_cfg["log-manager"]["token_expired_second"] }}'
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
       volumes:
         - name: pai-log
           hostPath:

src/log-manager/src/cleaner/entrypoint.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+set -o errexit
+set -o pipefail
+
+log_exist_time=30 # 30 day
+if [ -n "${LOG_EXIST_TIME}" ]; then
+  log_exist_time=${LOG_EXIST_TIME}
+fi
+
+cat > /etc/periodic/daily/remove_logs << EOF
+#!/bin/bash
+/usr/bin/pgrep -f ^find 2>&1 > /dev/null || find /usr/local/pai/logs/* -mtime +${log_exist_time} -type f -exec rm -fv {} \;
+EOF
+
+cat > /etc/periodic/weekly/remove_log_dir << EOF
+#!/bin/bash
+"/usr/bin/pgrep -f ^find 2>&1 > /dev/null || find /usr/local/pai/logs/* -mtime +${log_exist_time} -type d -empty -exec rmdir -v {} \;"
+EOF
+
+chmod a+x /etc/periodic/daily/remove_logs /etc/periodic/weekly/remove_log_dir
+
+echo "cron job added"
+
+crond -f -l 0
+