1 // SPDX-License-Identifier: GPL-2.0-only
4  * Copyright (c) 2013-2014, Intel Corporation.
85 	int cpu = event->cpu;  in bts_buffer_setup_aux()
86 	int node = (cpu == -1) ? cpu : cpu_to_node(cpu);  in bts_buffer_setup_aux()
108 	buf->nr_pages = nr_pages;  in bts_buffer_setup_aux()
109 	buf->nr_bufs = nbuf;  in bts_buffer_setup_aux()
110 	buf->snapshot = overwrite;  in bts_buffer_setup_aux()
111 	buf->data_pages = pages;  in bts_buffer_setup_aux()
112 	buf->real_size = size - size % BTS_RECORD_SIZE;  in bts_buffer_setup_aux()
114 	for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {  in bts_buffer_setup_aux()
119 		buf->buf[nbuf].page = page;  in bts_buffer_setup_aux()
120 		buf->buf[nbuf].offset = offset;  in bts_buffer_setup_aux()
121 		buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);  in bts_buffer_setup_aux()
122 		buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;  in bts_buffer_setup_aux()
123 		pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;  in bts_buffer_setup_aux()
124 		buf->buf[nbuf].size -= pad;  in bts_buffer_setup_aux()
140 	return buf->buf[idx].offset + buf->buf[idx].displacement;  in bts_buffer_offset()
148 	struct bts_phys *phys = &buf->buf[buf->cur_buf];  in bts_config_buffer()
149 	unsigned long index, thresh = 0, end = phys->size;  in bts_config_buffer()
150 	struct page *page = phys->page;  in bts_config_buffer()
152 	index = local_read(&buf->head);  in bts_config_buffer()
154 	if (!buf->snapshot) {  in bts_config_buffer()
155 		if (buf->end < phys->offset + buf_size(page))  in bts_config_buffer()
156 			end = buf->end - phys->offset - phys->displacement;  in bts_config_buffer()
158 		index -= phys->offset + phys->displacement;  in bts_config_buffer()
160 		if (end - index > BTS_SAFETY_MARGIN)  in bts_config_buffer()
161 			thresh = end - BTS_SAFETY_MARGIN;  in bts_config_buffer()
162 		else if (end - index > BTS_RECORD_SIZE)  in bts_config_buffer()
163 			thresh = end - BTS_RECORD_SIZE;  in bts_config_buffer()
168 	ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;  in bts_config_buffer()
169 	ds->bts_index = ds->bts_buffer_base + index;  in bts_config_buffer()
170 	ds->bts_absolute_maximum = ds->bts_buffer_base + end;  in bts_config_buffer()
171 	ds->bts_interrupt_threshold = !buf->snapshot  in bts_config_buffer()
172 		? ds->bts_buffer_base + thresh  in bts_config_buffer()
173 		: ds->bts_absolute_maximum + BTS_RECORD_SIZE;  in bts_config_buffer()
178 	unsigned long index = head - phys->offset;  in bts_buffer_pad_out()
180 	memset(page_address(phys->page) + index, 0, phys->size - index);  in bts_buffer_pad_out()
187 	struct bts_buffer *buf = perf_get_aux(&bts->handle);  in bts_update()
188 	unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;  in bts_update()
193 	head = index + bts_buffer_offset(buf, buf->cur_buf);  in bts_update()
194 	old = local_xchg(&buf->head, head);  in bts_update()
196 	if (!buf->snapshot) {  in bts_update()
200 		if (ds->bts_index >= ds->bts_absolute_maximum)  in bts_update()
201 			perf_aux_output_flag(&bts->handle,  in bts_update()
208 		local_add(head - old, &buf->data_size);  in bts_update()
210 		local_set(&buf->data_size, head);  in bts_update()
226  *  - is set when bts::handle::event is valid, that is, between
228  *  - is zero otherwise;
229  *  - is ordered against bts::handle::event with a compiler barrier.
235 	struct bts_buffer *buf = perf_get_aux(&bts->handle);  in __bts_event_start()
236 	u64 config = 0;  in __bts_event_start()  local
238 	if (!buf->snapshot)  in __bts_event_start()
239 		config |= ARCH_PERFMON_EVENTSEL_INT;  in __bts_event_start()
240 	if (!event->attr.exclude_kernel)  in __bts_event_start()
241 		config |= ARCH_PERFMON_EVENTSEL_OS;  in __bts_event_start()
242 	if (!event->attr.exclude_user)  in __bts_event_start()
243 		config |= ARCH_PERFMON_EVENTSEL_USR;  in __bts_event_start()
253 	/* INACTIVE/STOPPED -> ACTIVE */  in __bts_event_start()
254 	WRITE_ONCE(bts->state, BTS_STATE_ACTIVE);  in __bts_event_start()
256 	intel_pmu_enable_bts(config);  in __bts_event_start()
266 	buf = perf_aux_output_begin(&bts->handle, event);  in bts_event_start()
270 	if (bts_buffer_reset(buf, &bts->handle))  in bts_event_start()
273 	bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;  in bts_event_start()
274 	bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;  in bts_event_start()
275 	bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;  in bts_event_start()
278 	event->hw.state = 0;  in bts_event_start()
285 	perf_aux_output_end(&bts->handle, 0);  in bts_event_start()
288 	event->hw.state = PERF_HES_STOPPED;  in bts_event_start()
295 	/* ACTIVE -> INACTIVE(PMI)/STOPPED(->stop()) */  in __bts_event_stop()
296 	WRITE_ONCE(bts->state, state);  in __bts_event_stop()
310 	int state = READ_ONCE(bts->state);  in bts_event_stop()
316 		buf = perf_get_aux(&bts->handle);  in bts_event_stop()
318 	event->hw.state |= PERF_HES_STOPPED;  in bts_event_stop()
324 			if (buf->snapshot)  in bts_event_stop()
325 				bts->handle.head =  in bts_event_stop()
326 					local_xchg(&buf->data_size,  in bts_event_stop()
327 						   buf->nr_pages << PAGE_SHIFT);  in bts_event_stop()
328 			perf_aux_output_end(&bts->handle,  in bts_event_stop()
329 			                    local_xchg(&buf->data_size, 0));  in bts_event_stop()
332 		cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;  in bts_event_stop()
333 		cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;  in bts_event_stop()
334 		cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;  in bts_event_stop()
335 		cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;  in bts_event_stop()
342 	int state = READ_ONCE(bts->state);  in intel_bts_enable_local()
355 	if (bts->handle.event)  in intel_bts_enable_local()
356 		__bts_event_start(bts->handle.event);  in intel_bts_enable_local()
367 	if (READ_ONCE(bts->state) != BTS_STATE_ACTIVE)  in intel_bts_disable_local()
370 	if (bts->handle.event)  in intel_bts_disable_local()
371 		__bts_event_stop(bts->handle.event, BTS_STATE_INACTIVE);  in intel_bts_disable_local()
377 	unsigned long head, space, next_space, pad, gap, skip, wakeup;  in bts_buffer_reset()  local
382 	if (buf->snapshot)  in bts_buffer_reset()
385 	head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);  in bts_buffer_reset()
387 	phys = &buf->buf[buf->cur_buf];  in bts_buffer_reset()
388 	space = phys->offset + phys->displacement + phys->size - head;  in bts_buffer_reset()
390 	if (space > handle->size) {  in bts_buffer_reset()
391 		space = handle->size;  in bts_buffer_reset()
392 		space -= space % BTS_RECORD_SIZE;  in bts_buffer_reset()
396 		next_buf = buf->cur_buf + 1;  in bts_buffer_reset()
397 		if (next_buf >= buf->nr_bufs)  in bts_buffer_reset()
399 		next_phys = &buf->buf[next_buf];  in bts_buffer_reset()
400 		gap = buf_size(phys->page) - phys->displacement - phys->size +  in bts_buffer_reset()
401 		      next_phys->displacement;  in bts_buffer_reset()
402 		skip = pad + gap;  in bts_buffer_reset()
403 		if (handle->size >= skip) {  in bts_buffer_reset()
404 			next_space = next_phys->size;  in bts_buffer_reset()
405 			if (next_space + skip > handle->size) {  in bts_buffer_reset()
406 				next_space = handle->size - skip;  in bts_buffer_reset()
407 				next_space -= next_space % BTS_RECORD_SIZE;  in bts_buffer_reset()
412 				ret = perf_aux_output_skip(handle, skip);  in bts_buffer_reset()
418 				head = phys->offset + phys->displacement;  in bts_buffer_reset()
424 				buf->cur_buf = next_buf;  in bts_buffer_reset()
425 				local_set(&buf->head, head);  in bts_buffer_reset()
431 	wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -  in bts_buffer_reset()
432 		 handle->head;  in bts_buffer_reset()
435 		space -= space % BTS_RECORD_SIZE;  in bts_buffer_reset()
438 	buf->end = head + space;  in bts_buffer_reset()
442 	 * we hit absolute_maximum - see bts_update()  in bts_buffer_reset()
445 		return -ENOSPC;  in bts_buffer_reset()
452 	struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;  in intel_bts_interrupt()
454 	struct perf_event *event = bts->handle.event;  in intel_bts_interrupt()
457 	int err = -ENOSPC, handled = 0;  in intel_bts_interrupt()
463 	if (ds && (ds->bts_index >= ds->bts_interrupt_threshold))  in intel_bts_interrupt()
470 	if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)  in intel_bts_interrupt()
473 	buf = perf_get_aux(&bts->handle);  in intel_bts_interrupt()
478 	 * Skip snapshot counters: they don't use the interrupt, but  in intel_bts_interrupt()
482 	if (buf->snapshot)  in intel_bts_interrupt()
485 	old_head = local_read(&buf->head);  in intel_bts_interrupt()
489 	if (old_head == local_read(&buf->head))  in intel_bts_interrupt()
492 	perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));  in intel_bts_interrupt()
494 	buf = perf_aux_output_begin(&bts->handle, event);  in intel_bts_interrupt()
496 		err = bts_buffer_reset(buf, &bts->handle);  in intel_bts_interrupt()
499 		WRITE_ONCE(bts->state, BTS_STATE_STOPPED);  in intel_bts_interrupt()
507 			perf_aux_output_end(&bts->handle, 0);  in intel_bts_interrupt()
523 	struct hw_perf_event *hwc = &event->hw;  in bts_event_add()
525 	event->hw.state = PERF_HES_STOPPED;  in bts_event_add()
527 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))  in bts_event_add()
528 		return -EBUSY;  in bts_event_add()
530 	if (bts->handle.event)  in bts_event_add()
531 		return -EBUSY;  in bts_event_add()
535 		if (hwc->state & PERF_HES_STOPPED)  in bts_event_add()
536 			return -EINVAL;  in bts_event_add()
552 	if (event->attr.type != bts_pmu.type)  in bts_event_init()
553 		return -ENOENT;  in bts_event_init()
559 	 * to the user in a zero-copy fashion.  in bts_event_init()
561 	if (event->attr.exclude_kernel) {  in bts_event_init()
562 		ret = perf_allow_kernel(&event->attr);  in bts_event_init()
568 		return -EBUSY;  in bts_event_init()
576 	event->destroy = bts_event_destroy;  in bts_event_init()
588 		return -ENODEV;  in bts_init()
596 		 * However, since this driver supports per-CPU and per-task inherit  in bts_init()
605 		return -ENODEV;  in bts_init()
620 	return perf_pmu_register(&bts_pmu, "intel_bts", -1);  in bts_init()