-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathprofiling.yaml
294 lines (272 loc) · 12.8 KB
/
profiling.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
scripts:
run-pidstat:
# WAIT_FOR: which signal should it wait before starting
# WAIT_END: which signal should it wait before stopping
# FILE_ID: a name to identify the generated file if multiple runs
- js: ${{PIDSTAT_ENABLED}}
then:
- wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
- js: ${{HEROES_ENABLED}}
then:
- log: starting pidstat against ${{HOST.HEROES_REST_PID}}..
- sh: pidstat -p ${{HOST.HEROES_REST_PID}} ${{PIDSTAT_PARAMS}} > /tmp/HEROES_pidstat_${{FILE_ID:all}}.log & export HEROES_PIDSTAT_PID="$!"
- queue-download: /tmp/HEROES_pidstat_${{FILE_ID:all}}.log
- js: ${{VILLAINS_ENABLED}}
then:
- log: starting pidstat against ${{HOST.VILLAINS_REST_PID}}..
- sh: pidstat -p ${{HOST.VILLAINS_REST_PID}} ${{PIDSTAT_PARAMS}} > /tmp/VILLAINS_pidstat_${{FILE_ID:all}}.log & export VILLAINS_PIDSTAT_PID="$!"
- queue-download: /tmp/VILLAINS_pidstat_${{FILE_ID:all}}.log
- js: ${{LOCATIONS_ENABLED}}
then:
- log: starting pidstat against ${{HOST.LOCATIONS_GRPC_PID}}..
- sh: pidstat -p ${{HOST.LOCATIONS_GRPC_PID}} ${{PIDSTAT_PARAMS}} > /tmp/LOCATIONS_pidstat_${{FILE_ID:all}}.log & export LOCATIONS_PIDSTAT_PID="$!"
- queue-download: /tmp/LOCATIONS_pidstat_${{FILE_ID:all}}.log
- js: ${{FIGHTS_ENABLED}}
then:
- log: starting pidstat against ${{HOST.FIGHTS_REST_PID}}..
- sh: pidstat -p ${{HOST.FIGHTS_REST_PID}} ${{PIDSTAT_PARAMS}} > /tmp/FIGHTS_pidstat_${{FILE_ID:all}}.log & export FIGHTS_PIDSTAT_PID="$!"
- queue-download: /tmp/FIGHTS_pidstat_${{FILE_ID:all}}.log
- wait-for: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
then:
- sh: kill ${HEROES_PIDSTAT_PID} || echo "pidstat already stopped"
- sh: kill ${VILLAINS_PIDSTAT_PID} || echo "pidstat already stopped"
- sh: kill ${LOCATIONS_PIDSTAT_PID} || echo "pidstat already stopped"
- sh: kill ${FIGHTS_PIDSTAT_PID} || echo "pidstat already stopped"
# measure RSS
run-pmap:
# WAIT_FOR: which signal should it wait before starting
# WAIT_END: which signal should it wait before stopping
# FILE_ID: a name to identify the generated file if multiple runs
- js: ${{PMAP_ENABLED}}
then:
- wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
- js: ${{FIGHTS_ENABLED}}
then:
- queue-download: /tmp/FIGHTS_rss_${{FILE_ID:all}}.log
- repeat-until: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
then:
- sleep: 1s
- sh: echo -n "$(date +"%I:%M:%S %p") " >> /tmp/FIGHTS_rss_${{FILE_ID:all}}.log
- sh:
# NOTE: with docker this requires starting the containers with the same host user, this is now sets by default with
# --user "$(id -u):$(id -g)"
command: pmap -x ${{HOST.FIGHTS_REST_PID}} ${{PMAP_PARAMS:}} 2>/dev/null | grep total >> /tmp/FIGHTS_rss_${{FILE_ID:all}}.log
ignore-exit-code: true
run-strace:
# WAIT_FOR: which signal should it wait before starting
# WAIT_END: which signal should it wait before stopping
# FILE_ID: a name to identify the generated file if multiple runs
- js: ${{STRACE_ENABLED}}
then:
- wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
- js: ${{FIGHTS_ENABLED}}
then:
- log: starting strace against ${{HOST.FIGHTS_REST_PID}}..
- sh: strace -p ${{HOST.FIGHTS_REST_PID}} ${{STRACE_PARAMS:}} 2> /tmp/FIGHTS_strace_${{FILE_ID:all}}.log & export FIGHTS_STRACE_PID="$!"
- queue-download: /tmp/FIGHTS_strace_${{FILE_ID:all}}.log
- wait-for: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
then:
- sh: kill ${FIGHTS_STRACE_PID} || echo "strace already stopped"
run-perfstat:
# WAIT_FOR: which signal should it wait before starting
# WAIT_END: which signal should it wait before stopping
# PERFSTAT_SERVICE: service to monitor with perf stat FIGHTS, HEROES, VILLAINS or LOCATIONS
# FILE_ID: a name to identify the generated file if multiple runs
- js: ${{PERFSTAT_ENABLED}} && ${{${{PERFSTAT_SERVICE}}_ENABLED}}
then:
- wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
# - js: ${{FIGHTS_ENABLED}}
# then:
- queue-download: /tmp/${{PERFSTAT_SERVICE}}_perfstat_${{FILE_ID:all}}.log
- log: starting perf stat against ${{HOST.${{PERFSTAT_SERVICE}}_REST_PID}}..
- sh: perf stat -d -p ${{HOST.${{PERFSTAT_SERVICE}}_REST_PID}} ${{PERFSTAT_PARAMS:}} -o /tmp/${{PERFSTAT_SERVICE}}_perfstat_${{FILE_ID:all}}.log
silent: true # Prevent the 'Nanny found idle' warnings
watch:
- regex: Problems finding threads of monitor
then:
- abort: Failed to start perf, unable to find provided pid ${{HOST.${{PERFSTAT_SERVICE}}_REST_PID}}
on-signal:
${{WAIT_END:HF_BENCHMARK_TERMINATED}}:
- log: Stopping perf
- ctrlC
run-vmstat:
# WAIT_FOR: which signal should it wait before starting
# WAIT_END: which signal should it wait before stopping
- js: ${{VMSTAT_ENABLED}}
then:
- wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
- log: starting vmstat..
- sh: vmstat ${{VMSTAT_PARAMS}} > /tmp/vmstat.log & export VMSTAT_PID="$!"
- queue-download: /tmp/vmstat.log
- wait-for: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
then:
- sh: kill ${VMSTAT_PID} || echo "vmstat already stopped"
run-mpstat:
# WAIT_FOR: which signal should it wait before starting
# WAIT_END: which signal should it wait before stopping
- js: ${{MPSTAT_ENABLED}}
then:
- wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
- log: starting mpstat..
- sh: mpstat ${{MPSTAT_PARAMS}} > /tmp/mpstat.log & export MPSTAT_PID="$!"
- queue-download: /tmp/mpstat.log
- wait-for: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
then:
- sh: kill ${MPSTAT_PID} || echo "mpstat already stopped"
# application profiling
app-prof-setup:
- js: ${{APP_PROF_ENABLED}}
then:
- sh: ${{SUDOER}} sh -c "echo 1 > /proc/sys/kernel/perf_event_paranoid" || echo "Unable to set perf_event_paranoid"
- sh: ${{SUDOER}} sh -c "echo 0 > /proc/sys/kernel/kptr_restrict" || echo "Unable to unset kptr_restrict"
- js: ${{IS_NATIVE}}
else:
- script: asprof-setup
# async profiler
asprof-setup:
- js: ${{APP_PROF_ENABLED}}
then:
# override container runtimes to include asprof volumes
- set-state: CONTAINER_RUNTIME_PARAMS ${{CONTAINER_RUNTIME_PARAMS}} -v ${{ASPROF_DIR}}:${{ASPROF_DIR}}:rw,Z -v ${{ASPROF_OUT_DIR}}:${{ASPROF_OUT_DIR}}:rw,Z
- sh: if [ -d ${{ASPROF_DIR}} ]; then rm -rf ${{ASPROF_DIR}}; fi
- sh: curl -L ${{ASPROF_RELEASE_URL}} > /tmp/async-profiler.tgz
- sh: tar -xvf /tmp/async-profiler.tgz -C /tmp
- sh: mv /tmp/async-profiler-${{ASPROF_VERSION:3.0}}-linux-x64 ${{ASPROF_DIR:/tmp/async-profiler}}
# cleanup downloaded tar
- sh: rm -rf /tmp/async-profiler.tgz
- set-state: RUN.ASPROF_EXEC ${{ASPROF_DIR:/tmp/async-profiler}}/bin/asprof
- sh: if [ -d ${{ASPROF_OUT_DIR}} ]; then rm -rf ${{ASPROF_OUT_DIR}}; fi
- sh: mkdir -p ${{ASPROF_OUT_DIR}} && chmod 777 ${{ASPROF_OUT_DIR}} || echo "Cannot create ${{ASPROF_OUT_DIR}} directory"
run-app-prof:
- js: ${{APP_PROF_ENABLED}}
then:
- js: ${{IS_NATIVE}}
then:
- script: run-perfrecord
else:
- script: run-asprof
run-perfrecord:
# WAIT_FOR: which signal should it wait before starting
# WAIT_END: which signal should it wait before stopping
# APP_PID_STATE: the pid of the JVM process to profile
# APP_SERVICE: service to monitor FIGHTS, HEROES, VILLAINS or LOCATIONS
# FILE_ID: a name to identify the generated file if multiple runs
- wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
- log: Starting application level profiling for native ${{${{APP_PID_STATE}}}}...
- sh:
command: ${{SUDOER}} ls -la /work
ignore-exit-code: true
- sh: ${{SUDOER}} perf record ${{PERFRECORD_PARAMS}} -o /tmp/${{APP_SERVICE}}_perfrecord.data -p ${{${{APP_PID_STATE}}}}
on-signal:
# stop perf when the HF load terminates
${{WAIT_END:HF_BENCHMARK_TERMINATED}}:
- ctrlC
- sh: ${{SUDOER}} perf script -i /tmp/${{APP_SERVICE}}_perfrecord.data >> /tmp/${{APP_SERVICE}}_perfrecord-script.txt
# - sh: ${{SUDOER}} perf report -i /tmp/${{APP_SERVICE}}_perfrecord.data >> /tmp/${{APP_SERVICE}}_perfrecord-report.txt
- queue-download: /tmp/${{APP_SERVICE}}_perfrecord.data
- queue-download: /tmp/${{APP_SERVICE}}_perfrecord-script.txt
# - queue-download: /tmp/${{APP_SERVICE}}_perfrecord-report.txt
# remove the /work emulation if present
- sh: if [ -d /work ]; then rm -rf /work || echo "Cannot remove /work directory"; fi
run-asprof:
# Relies on existing async-profiler exectuable located at ASPROF_EXEC
# APP_PID_STATE: the pid of the JVM process to profile
# APP_SERVICE: service to monitor FIGHTS, HEROES, VILLAINS or LOCATIONS
# FILE_ID: a name to identify the generated file if multiple runs
- wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
- log: Starting application level profiling for JVM ${{${{APP_PID_STATE}}}}...
- sh: ${{SUDOER}} ${{ASPROF_EXEC:/tmp/async-profiler/bin/asprof}} start -e ${{ASPROF_EVENT}} --fdtransfer ${{ASPROF_PARAMS:}} ${{${{APP_PID_STATE}}}}
- log: Async profiler started, waiting for ${{WAIT_END:HF_BENCHMARK_TERMINATED}} stop signal...
- wait-for: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
- sh: ${{SUDOER}} ${{ASPROF_EXEC:/tmp/async-profiler/bin/asprof}} stop -f ${{ASPROF_OUT_DIR}}/${{APP_SERVICE:service}}_asprof_${{FILE_ID:all}} -o ${{ASPROF_FORMAT:flamegraph}} --title '${{APP_SERVICE:service}} ${{ASPROF_EVENT}} profiling' ${{${{APP_PID_STATE}}}}
- log: Stopped application level profiling
# Rename the output file based on the provided format file extension
- set-state: FILE_EXTENSION ${{= ${{ASPROF_FORMATS}}['${{ASPROF_FORMAT:flamegraph}}'] }}
- sh: mv ${{ASPROF_OUT_DIR}}/${{APP_SERVICE:service}}_asprof_${{FILE_ID:all}} ${{ASPROF_OUT_DIR}}/${{APP_SERVICE:service}}_asprof_${{FILE_ID:all}}.${{FILE_EXTENSION}}
- queue-download: ${{ASPROF_OUT_DIR}}/${{APP_SERVICE:service}}_asprof_${{FILE_ID:all}}.${{FILE_EXTENSION}}
# parse the metrics from the generated profiling files and set them to
# qDup states so that they can easily be exported
# RSS at the end
# RSS (min, max, mean)
# CPU usage (min, max, mean)
# vmstat? mpstat?
export-metrics:
# stats for pidstat
- js: ${{PIDSTAT_ENABLED}}
then:
- wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
- js: ${{HEROES_ENABLED}}
then:
- script: parse-pidstat-results
with:
PIDSTAT_FILE: /tmp/HEROES_pidstat_${{FILE_ID:all}}.log
MAX_STATE: ${{FILE_ID:all}}_HEROES_CPU_USAGE_MAX
AVG_STATE: ${{FILE_ID:all}}_HEROES_CPU_USAGE_AVG
- js: ${{VILLAINS_ENABLED}}
then:
- script: parse-pidstat-results
with:
PIDSTAT_FILE: /tmp/VILLAINS_pidstat_${{FILE_ID:all}}.log
MAX_STATE: ${{FILE_ID:all}}_VILLAINS_CPU_USAGE_MAX
AVG_STATE: ${{FILE_ID:all}}_VILLAINS_CPU_USAGE_AVG
- js: ${{LOCATIONS_ENABLED}}
then:
- script: parse-pidstat-results
with:
PIDSTAT_FILE: /tmp/LOCATIONS_pidstat_${{FILE_ID:all}}.log
MAX_STATE: ${{FILE_ID:all}}_LOCATIONS_CPU_USAGE_MAX
AVG_STATE: ${{FILE_ID:all}}_LOCATIONS_CPU_USAGE_AVG
- js: ${{FIGHTS_ENABLED}}
then:
- script: parse-pidstat-results
with:
PIDSTAT_FILE: /tmp/FIGHTS_pidstat_${{FILE_ID:all}}.log
MAX_STATE: ${{FILE_ID:all}}_FIGHTS_CPU_USAGE_MAX
AVG_STATE: ${{FILE_ID:all}}_FIGHTS_CPU_USAGE_AVG
# rss
- js: ${{PMAP_ENABLED}} && ${{FIGHTS_ENABLED}}
then:
- script: parse-rss-results
with:
RSS_FILE: /tmp/FIGHTS_rss_${{FILE_ID:all}}.log
MAX_STATE: ${{FILE_ID:all}}_FIGHTS_RSS_MAX
AVG_STATE: ${{FILE_ID:all}}_FIGHTS_RSS_AVG
cleanup-profiling:
- sh:
command: rm -rf /tmp/vmstat.log /tmp/mpstat.log /tmp/*_rss*.log /tmp/*_pidstat*.log /tmp/*_perfstat_${{FILE_ID:all}}.log
ignore-exit-code: true
- sh:
command: if [ -d ${{ASPROF_DIR}} ]; then rm -rf ${{ASPROF_DIR}}; fi
ignore-exit-code: true
- sh:
command: if [ -d ${{ASPROF_OUT_DIR}} ]; then rm -rf ${{ASPROF_OUT_DIR}}; fi
ignore-exit-code: true
states:
SUDOER:
PIDSTAT_ENABLED: true
PIDSTAT_PARAMS: "1"
PMAP_ENABLED: true
PMAP_PARAMS:
STRACE_ENABLED: false
STRACE_PARAMS: -s 99 -ff
PERFSTAT_ENABLED: true
PERFSTAT_PARAMS:
VMSTAT_ENABLED: true
VMSTAT_PARAMS: "1"
MPSTAT_ENABLED: true
MPSTAT_PARAMS: "-P ALL 1"
# in according to the stack it will use either perf record or asprof
APP_PROF_ENABLED: false
ASPROF_RELEASE_URL: https://github.com/async-profiler/async-profiler/releases/download/v${{ASPROF_VERSION:3.0}}/async-profiler-${{ASPROF_VERSION:3.0}}-linux-x64.tar.gz
ASPROF_VERSION: "3.0"
ASPROF_DIR: /tmp/async-profiler
ASPROF_PARAMS:
ASPROF_EVENT: cpu
ASPROF_FORMAT: flamegraph
ASPROF_OUT_DIR: /tmp/async-profiler-output
ASPROF_FORMATS: |
{
"jfr": "jfr",
"flamegraph": "html"
}
PERFRECORD_PARAMS: -e cpu-clock -F 100 --call-graph dwarf