1 // SPDX-License-Identifier: GPL-2.0
2 #include "util/bpf_counter.h"
3 #include "util/debug.h"
4 #include "util/evsel.h"
5 #include "util/evlist.h"
6 #include "util/off_cpu.h"
7 #include "util/perf-hooks.h"
8 #include "util/record.h"
9 #include "util/session.h"
10 #include "util/target.h"
11 #include "util/cpumap.h"
12 #include "util/thread_map.h"
13 #include "util/cgroup.h"
14 #include "util/strlist.h"
15 #include <bpf/bpf.h>
16
17 #include "bpf_skel/off_cpu.skel.h"
18
19 #define MAX_STACKS 32
20 #define MAX_PROC 4096
21 /* we don't need actual timestamp, just want to put the samples at last */
22 #define OFF_CPU_TIMESTAMP (~0ull << 32)
23
24 static struct off_cpu_bpf *skel;
25
26 struct off_cpu_key {
27 u32 pid;
28 u32 tgid;
29 u32 stack_id;
30 u32 state;
31 u64 cgroup_id;
32 };
33
34 union off_cpu_data {
35 struct perf_event_header hdr;
36 u64 array[1024 / sizeof(u64)];
37 };
38
off_cpu_config(struct evlist * evlist)39 static int off_cpu_config(struct evlist *evlist)
40 {
41 struct evsel *evsel;
42 struct perf_event_attr attr = {
43 .type = PERF_TYPE_SOFTWARE,
44 .config = PERF_COUNT_SW_BPF_OUTPUT,
45 .size = sizeof(attr), /* to capture ABI version */
46 };
47 char *evname = strdup(OFFCPU_EVENT);
48
49 if (evname == NULL)
50 return -ENOMEM;
51
52 evsel = evsel__new(&attr);
53 if (!evsel) {
54 free(evname);
55 return -ENOMEM;
56 }
57
58 evsel->core.attr.freq = 1;
59 evsel->core.attr.sample_period = 1;
60 /* off-cpu analysis depends on stack trace */
61 evsel->core.attr.sample_type = PERF_SAMPLE_CALLCHAIN;
62
63 evlist__add(evlist, evsel);
64
65 free(evsel->name);
66 evsel->name = evname;
67
68 return 0;
69 }
70
off_cpu_start(void * arg)71 static void off_cpu_start(void *arg)
72 {
73 struct evlist *evlist = arg;
74
75 /* update task filter for the given workload */
76 if (skel->rodata->has_task && skel->rodata->uses_tgid &&
77 perf_thread_map__pid(evlist->core.threads, 0) != -1) {
78 int fd;
79 u32 pid;
80 u8 val = 1;
81
82 fd = bpf_map__fd(skel->maps.task_filter);
83 pid = perf_thread_map__pid(evlist->core.threads, 0);
84 bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
85 }
86
87 skel->bss->enabled = 1;
88 }
89
off_cpu_finish(void * arg __maybe_unused)90 static void off_cpu_finish(void *arg __maybe_unused)
91 {
92 skel->bss->enabled = 0;
93 off_cpu_bpf__destroy(skel);
94 }
95
96 /* v5.18 kernel added prev_state arg, so it needs to check the signature */
check_sched_switch_args(void)97 static void check_sched_switch_args(void)
98 {
99 struct btf *btf = btf__load_vmlinux_btf();
100 const struct btf_type *t1, *t2, *t3;
101 u32 type_id;
102
103 if (!btf) {
104 pr_debug("Missing btf, check if CONFIG_DEBUG_INFO_BTF is enabled\n");
105 goto cleanup;
106 }
107
108 type_id = btf__find_by_name_kind(btf, "btf_trace_sched_switch",
109 BTF_KIND_TYPEDEF);
110 if ((s32)type_id < 0)
111 goto cleanup;
112
113 t1 = btf__type_by_id(btf, type_id);
114 if (t1 == NULL)
115 goto cleanup;
116
117 t2 = btf__type_by_id(btf, t1->type);
118 if (t2 == NULL || !btf_is_ptr(t2))
119 goto cleanup;
120
121 t3 = btf__type_by_id(btf, t2->type);
122 /* btf_trace func proto has one more argument for the context */
123 if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 5) {
124 /* new format: pass prev_state as 4th arg */
125 skel->rodata->has_prev_state = true;
126 }
127 cleanup:
128 btf__free(btf);
129 }
130
off_cpu_prepare(struct evlist * evlist,struct target * target,struct record_opts * opts)131 int off_cpu_prepare(struct evlist *evlist, struct target *target,
132 struct record_opts *opts)
133 {
134 int err, fd, i;
135 int ncpus = 1, ntasks = 1, ncgrps = 1;
136 struct strlist *pid_slist = NULL;
137 struct str_node *pos;
138
139 if (off_cpu_config(evlist) < 0) {
140 pr_err("Failed to config off-cpu BPF event\n");
141 return -1;
142 }
143
144 skel = off_cpu_bpf__open();
145 if (!skel) {
146 pr_err("Failed to open off-cpu BPF skeleton\n");
147 return -1;
148 }
149
150 /* don't need to set cpu filter for system-wide mode */
151 if (target->cpu_list) {
152 ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus);
153 bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus);
154 skel->rodata->has_cpu = 1;
155 }
156
157 if (target->pid) {
158 pid_slist = strlist__new(target->pid, NULL);
159 if (!pid_slist) {
160 pr_err("Failed to create a strlist for pid\n");
161 return -1;
162 }
163
164 ntasks = 0;
165 strlist__for_each_entry(pos, pid_slist) {
166 char *end_ptr;
167 int pid = strtol(pos->s, &end_ptr, 10);
168
169 if (pid == INT_MIN || pid == INT_MAX ||
170 (*end_ptr != '\0' && *end_ptr != ','))
171 continue;
172
173 ntasks++;
174 }
175
176 if (ntasks < MAX_PROC)
177 ntasks = MAX_PROC;
178
179 bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
180 skel->rodata->has_task = 1;
181 skel->rodata->uses_tgid = 1;
182 } else if (target__has_task(target)) {
183 ntasks = perf_thread_map__nr(evlist->core.threads);
184 bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
185 skel->rodata->has_task = 1;
186 } else if (target__none(target)) {
187 bpf_map__set_max_entries(skel->maps.task_filter, MAX_PROC);
188 skel->rodata->has_task = 1;
189 skel->rodata->uses_tgid = 1;
190 }
191
192 if (evlist__first(evlist)->cgrp) {
193 ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */
194 bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps);
195
196 if (!cgroup_is_v2("perf_event"))
197 skel->rodata->uses_cgroup_v1 = true;
198 skel->rodata->has_cgroup = 1;
199 }
200
201 if (opts->record_cgroup) {
202 skel->rodata->needs_cgroup = true;
203
204 if (!cgroup_is_v2("perf_event"))
205 skel->rodata->uses_cgroup_v1 = true;
206 }
207
208 set_max_rlimit();
209 check_sched_switch_args();
210
211 err = off_cpu_bpf__load(skel);
212 if (err) {
213 pr_err("Failed to load off-cpu skeleton\n");
214 goto out;
215 }
216
217 if (target->cpu_list) {
218 u32 cpu;
219 u8 val = 1;
220
221 fd = bpf_map__fd(skel->maps.cpu_filter);
222
223 for (i = 0; i < ncpus; i++) {
224 cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu;
225 bpf_map_update_elem(fd, &cpu, &val, BPF_ANY);
226 }
227 }
228
229 if (target->pid) {
230 u8 val = 1;
231
232 fd = bpf_map__fd(skel->maps.task_filter);
233
234 strlist__for_each_entry(pos, pid_slist) {
235 char *end_ptr;
236 u32 tgid;
237 int pid = strtol(pos->s, &end_ptr, 10);
238
239 if (pid == INT_MIN || pid == INT_MAX ||
240 (*end_ptr != '\0' && *end_ptr != ','))
241 continue;
242
243 tgid = pid;
244 bpf_map_update_elem(fd, &tgid, &val, BPF_ANY);
245 }
246 } else if (target__has_task(target)) {
247 u32 pid;
248 u8 val = 1;
249
250 fd = bpf_map__fd(skel->maps.task_filter);
251
252 for (i = 0; i < ntasks; i++) {
253 pid = perf_thread_map__pid(evlist->core.threads, i);
254 bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
255 }
256 }
257
258 if (evlist__first(evlist)->cgrp) {
259 struct evsel *evsel;
260 u8 val = 1;
261
262 fd = bpf_map__fd(skel->maps.cgroup_filter);
263
264 evlist__for_each_entry(evlist, evsel) {
265 struct cgroup *cgrp = evsel->cgrp;
266
267 if (cgrp == NULL)
268 continue;
269
270 if (!cgrp->id && read_cgroup_id(cgrp) < 0) {
271 pr_err("Failed to read cgroup id of %s\n",
272 cgrp->name);
273 goto out;
274 }
275
276 bpf_map_update_elem(fd, &cgrp->id, &val, BPF_ANY);
277 }
278 }
279
280 err = off_cpu_bpf__attach(skel);
281 if (err) {
282 pr_err("Failed to attach off-cpu BPF skeleton\n");
283 goto out;
284 }
285
286 if (perf_hooks__set_hook("record_start", off_cpu_start, evlist) ||
287 perf_hooks__set_hook("record_end", off_cpu_finish, evlist)) {
288 pr_err("Failed to attach off-cpu skeleton\n");
289 goto out;
290 }
291
292 return 0;
293
294 out:
295 off_cpu_bpf__destroy(skel);
296 return -1;
297 }
298
off_cpu_write(struct perf_session * session)299 int off_cpu_write(struct perf_session *session)
300 {
301 int bytes = 0, size;
302 int fd, stack;
303 u64 sample_type, val, sid = 0;
304 struct evsel *evsel;
305 struct perf_data_file *file = &session->data->file;
306 struct off_cpu_key prev, key;
307 union off_cpu_data data = {
308 .hdr = {
309 .type = PERF_RECORD_SAMPLE,
310 .misc = PERF_RECORD_MISC_USER,
311 },
312 };
313 u64 tstamp = OFF_CPU_TIMESTAMP;
314
315 skel->bss->enabled = 0;
316
317 evsel = evlist__find_evsel_by_str(session->evlist, OFFCPU_EVENT);
318 if (evsel == NULL) {
319 pr_err("%s evsel not found\n", OFFCPU_EVENT);
320 return 0;
321 }
322
323 sample_type = evsel->core.attr.sample_type;
324
325 if (sample_type & ~OFFCPU_SAMPLE_TYPES) {
326 pr_err("not supported sample type: %llx\n",
327 (unsigned long long)sample_type);
328 return -1;
329 }
330
331 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) {
332 if (evsel->core.id)
333 sid = evsel->core.id[0];
334 }
335
336 fd = bpf_map__fd(skel->maps.off_cpu);
337 stack = bpf_map__fd(skel->maps.stacks);
338 memset(&prev, 0, sizeof(prev));
339
340 while (!bpf_map_get_next_key(fd, &prev, &key)) {
341 int n = 1; /* start from perf_event_header */
342 int ip_pos = -1;
343
344 bpf_map_lookup_elem(fd, &key, &val);
345
346 if (sample_type & PERF_SAMPLE_IDENTIFIER)
347 data.array[n++] = sid;
348 if (sample_type & PERF_SAMPLE_IP) {
349 ip_pos = n;
350 data.array[n++] = 0; /* will be updated */
351 }
352 if (sample_type & PERF_SAMPLE_TID)
353 data.array[n++] = (u64)key.pid << 32 | key.tgid;
354 if (sample_type & PERF_SAMPLE_TIME)
355 data.array[n++] = tstamp;
356 if (sample_type & PERF_SAMPLE_ID)
357 data.array[n++] = sid;
358 if (sample_type & PERF_SAMPLE_CPU)
359 data.array[n++] = 0;
360 if (sample_type & PERF_SAMPLE_PERIOD)
361 data.array[n++] = val;
362 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
363 int len = 0;
364
365 /* data.array[n] is callchain->nr (updated later) */
366 data.array[n + 1] = PERF_CONTEXT_USER;
367 data.array[n + 2] = 0;
368
369 bpf_map_lookup_elem(stack, &key.stack_id, &data.array[n + 2]);
370 while (data.array[n + 2 + len])
371 len++;
372
373 /* update length of callchain */
374 data.array[n] = len + 1;
375
376 /* update sample ip with the first callchain entry */
377 if (ip_pos >= 0)
378 data.array[ip_pos] = data.array[n + 2];
379
380 /* calculate sample callchain data array length */
381 n += len + 2;
382 }
383 if (sample_type & PERF_SAMPLE_CGROUP)
384 data.array[n++] = key.cgroup_id;
385
386 size = n * sizeof(u64);
387 data.hdr.size = size;
388 bytes += size;
389
390 if (perf_data_file__write(file, &data, size) < 0) {
391 pr_err("failed to write perf data, error: %m\n");
392 return bytes;
393 }
394
395 prev = key;
396 /* increase dummy timestamp to sort later samples */
397 tstamp++;
398 }
399 return bytes;
400 }
401