Skip to content
This repository has been archived by the owner on Jun 6, 2024. It is now read-only.

YARN and HDFS log persistence #2244

Merged
merged 11 commits into from
Mar 6, 2019
Merged

YARN and HDFS log persistence #2244

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
c29c15e
rm log persist
mzmssg Feb 15, 2019
b41017b
change log dir to host
mzmssg Feb 18, 2019
cfce71f
persist nm log to host
mzmssg Mar 1, 2019
24d13c6
resolve conflict
mzmssg Mar 1, 2019
89cf589
resolve conflict
mzmssg Mar 1, 2019
b945300
Merge branch 'master' into zimiao/persist_log
mzmssg Mar 4, 2019
5551a1b
Merge branch 'master' into zimiao/persist_log
mzmssg Mar 5, 2019
a0bb256
persist namenode log
mzmssg Mar 5, 2019
7a917b8
persist data node log
mzmssg Mar 5, 2019
7b5f8c8
add comments
mzmssg Mar 5, 2019
cbf6b36
move log path to common pai storage
mzmssg Mar 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/hadoop-data-node/deploy/hadoop-data-node.yaml.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ spec:
name: host-confg-volume
- mountPath: /var/lib/hadoopdata
name: hadoop-tmp-storage
- mountPath: /var/log/hadoop
name: log-dir
readinessProbe:
exec:
command:
Expand Down Expand Up @@ -77,6 +79,11 @@ spec:
value: datanode-start-service.sh
- name: HADOOP_DATANODE_DATA_DIR
value: {{ mount_points|join(",") }}
# Rolling File Appender, by default it keeps at most 256M*20=5G logs.
- name: HADOOP_ROOT_LOGGER
value: INFO,console,RFA
- name: HADOOP_LOG_DIR
value: /var/log/hadoop
- name: POD_IP
valueFrom:
fieldRef:
Expand All @@ -98,6 +105,9 @@ spec:
- name: host-confg-volume
configMap:
name: host-configuration
- name: log-dir
hostPath:
path: /var/log/data-node
tolerations:
- key: node.kubernetes.io/memory-pressure
operator: "Exists"
Expand Down
10 changes: 10 additions & 0 deletions src/hadoop-name-node/deploy/hadoop-name-node.yaml.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ spec:
name: hadoop-name-node-config-volume
- mountPath: /var/lib/hadoopdata
name: hadoop-tmp-storage
- mountPath: /var/log/hadoop
name: log-dir
readinessProbe:
exec:
command:
Expand All @@ -57,6 +59,11 @@ spec:
value: namenode-generate-script.sh
- name: START_SERVICE
value: namenode-start-service.sh
# Rolling File Appender, by default it keeps at most 256M*20=5G logs.
- name: HADOOP_ROOT_LOGGER
value: INFO,console,RFA
- name: HADOOP_LOG_DIR
value: /var/log/hadoop
{%- if cluster_cfg['cluster']['common']['qos-switch'] == "true" %}
resources:
limits:
Expand All @@ -74,3 +81,6 @@ spec:
- name: hadoop-tmp-storage
hostPath:
path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/hadooptmp/namenode
- name: log-dir
hostPath:
path: /var/log/name-node
10 changes: 10 additions & 0 deletions src/hadoop-node-manager/deploy/hadoop-node-manager.yaml.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ spec:
name: host-confg-volume
- mountPath: /var/lib/hadoopdata
name: hadoop-tmp-storage
- mountPath: /var/log/hadoop
name: log-dir
readinessProbe:
exec:
command:
Expand Down Expand Up @@ -107,6 +109,11 @@ spec:
value: "3072"
- name: NV_DRIVER
value: /var/drivers/nvidia/current
# Rolling File Appender, by default it keeps at most 256M*20=5G logs.
- name: YARN_ROOT_LOGGER
value: INFO,console,RFA
- name: YARN_LOG_DIR
value: /var/log/hadoop
- name: POD_IP
valueFrom:
fieldRef:
Expand Down Expand Up @@ -144,6 +151,9 @@ spec:
- name: hadoop-tmp-storage
hostPath:
path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/hadooptmp/nodemanager
- name: log-dir
hostPath:
path: /var/log/node-manager
tolerations:
- key: node.kubernetes.io/memory-pressure
operator: "Exists"
Expand Down
10 changes: 10 additions & 0 deletions src/hadoop-resource-manager/deploy/hadoop-resource-manager.yaml.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ spec:
name: yarn-resourcemanager-storage
- mountPath: /var/lib/hadoopdata
name: hadoop-tmp-storage
- mountPath: /var/log/hadoop
name: log-dir
- mountPath: /exclude-path
name: hadoop-resource-manager-exclude-nodes
readinessProbe:
Expand All @@ -78,6 +80,11 @@ spec:
value: resourcemanager-generate-script.sh
- name: START_SERVICE
value: resourcemanager-start-service.sh
# Rolling File Appender, by default it keeps at most 256M*20=5G logs.
- name: YARN_ROOT_LOGGER
value: INFO,console,RFA
- name: YARN_LOG_DIR
value: /var/log/hadoop
{%- if cluster_cfg['cluster']['common']['qos-switch'] == "true" %}
resources:
limits:
Expand Down Expand Up @@ -115,6 +122,9 @@ spec:
- name: hadoop-tmp-storage
hostPath:
path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/hadooptmp/resourcemanager
- name: log-dir
hostPath:
path: /var/log/resource-manager
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

{{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/pai-service-log/resourcemanager?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

- name: hadoop-resource-manager-exclude-nodes
configMap:
name: exclude-file