255 static int gru_get_cpu_resources(int dsr_bytes, void **cb, void **dsr)  in gru_get_cpu_resources()  argument
263 	*cb = bs->kernel_cb + lcpu * GRU_HANDLE_STRIDE;  in gru_get_cpu_resources()
271 static void gru_free_cpu_resources(void *cb, void *dsr)  in gru_free_cpu_resources()  argument
357 void gru_lock_async_resource(unsigned long han,  void **cb, void **dsr)  in gru_lock_async_resource()  argument
365 	if (cb)  in gru_lock_async_resource()
366 		*cb = bs->kernel_cb + ncpus * GRU_HANDLE_STRIDE;  in gru_lock_async_resource()
385 int gru_get_cb_exception_detail(void *cb,  in gru_get_cb_exception_detail()  argument
404 		off = cb - kgts->ts_gru->gs_gru_base_vaddr;  in gru_get_cb_exception_detail()
410 	cbrnum = thread_cbr_number(kgts, get_cb_number(cb));  in gru_get_cb_exception_detail()
411 	cbe = get_cbe(GRUBASE(cb), cbrnum);  in gru_get_cb_exception_detail()
423 static char *gru_get_cb_exception_detail_str(int ret, void *cb,  in gru_get_cb_exception_detail_str()  argument
426 	struct gru_control_block_status *gen = cb;  in gru_get_cb_exception_detail_str()
430 		gru_get_cb_exception_detail(cb, &excdet);  in gru_get_cb_exception_detail_str()
451 static int gru_retry_exception(void *cb)  in gru_retry_exception()  argument
453 	struct gru_control_block_status *gen = cb;  in gru_retry_exception()
460 		if (gru_get_cb_message_queue_substatus(cb))  in gru_retry_exception()
462 		gru_get_cb_exception_detail(cb, &excdet);  in gru_retry_exception()
474 int gru_check_status_proc(void *cb)  in gru_check_status_proc()  argument
476 	struct gru_control_block_status *gen = cb;  in gru_check_status_proc()
481 		ret = gru_retry_exception(cb);  in gru_check_status_proc()
487 int gru_wait_proc(void *cb)  in gru_wait_proc()  argument
489 	struct gru_control_block_status *gen = cb;  in gru_wait_proc()
494 		ret = gru_retry_exception(cb);  in gru_wait_proc()
499 static void gru_abort(int ret, void *cb, char *str)  in gru_abort()  argument
504 	      gru_get_cb_exception_detail_str(ret, cb, buf, sizeof(buf)));  in gru_abort()
507 void gru_wait_abort_proc(void *cb)  in gru_wait_abort_proc()  argument
511 	ret = gru_wait_proc(cb);  in gru_wait_abort_proc()
513 		gru_abort(ret, cb, "gru_wait_abort");  in gru_wait_abort_proc()
577 static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd,  in send_noop_message()  argument
589 	gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA);  in send_noop_message()
590 	ret = gru_wait(cb);  in send_noop_message()
593 		substatus = gru_get_cb_message_queue_substatus(cb);  in send_noop_message()
613 			m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);  in send_noop_message()
614 			gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1,  in send_noop_message()
616 			if (gru_wait(cb) == CBS_IDLE)  in send_noop_message()
635 static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd,  in send_message_queue_full()  argument
644 	avalue = gru_get_amo_value(cb);  in send_message_queue_full()
645 	head = gru_get_amo_value_head(cb);  in send_message_queue_full()
646 	limit = gru_get_amo_value_limit(cb);  in send_message_queue_full()
657 	gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA);  in send_message_queue_full()
658 	if (gru_wait(cb) != CBS_IDLE)  in send_message_queue_full()
660 	if (!gru_get_amo_value(cb)) {  in send_message_queue_full()
667 		if (send_noop_message(cb, mqd, mesg)) {  in send_message_queue_full()
668 			gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half),  in send_message_queue_full()
670 			if (gru_wait(cb) != CBS_IDLE)  in send_message_queue_full()
679 	gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue,  in send_message_queue_full()
681 	if (gru_wait(cb) != CBS_IDLE)  in send_message_queue_full()
685 	if (gru_get_amo_value(cb) != avalue) {  in send_message_queue_full()
687 		gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW,  in send_message_queue_full()
689 		if (gru_wait(cb) != CBS_IDLE)  in send_message_queue_full()
704 static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd,  in send_message_put_nacked()  argument
710 	m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);  in send_message_put_nacked()
712 		gru_vset(cb, m, 0, XTYPE_CL, lines, 1, IMA);  in send_message_put_nacked()
713 		if (gru_wait(cb) != CBS_IDLE)  in send_message_put_nacked()
716 	gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA);  in send_message_put_nacked()
717 	if (gru_wait(cb) != CBS_IDLE)  in send_message_put_nacked()
732 		ret = send_noop_message(cb, mqd, mesg);  in send_message_put_nacked()
752 static int send_message_failure(void *cb, struct gru_message_queue_desc *mqd,  in send_message_failure()  argument
757 	substatus = gru_get_cb_message_queue_substatus(cb);  in send_message_failure()
769 		ret = send_message_queue_full(cb, mqd, mesg, lines);  in send_message_failure()
777 		ret = send_message_put_nacked(cb, mqd, mesg, lines);  in send_message_failure()
798 	void *cb;  in gru_send_message_gpa()  local
806 	if (gru_get_cpu_resources(bytes, &cb, &dsr))  in gru_send_message_gpa()
819 		gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), clines, IMA);  in gru_send_message_gpa()
820 		istatus = gru_wait(cb);  in gru_send_message_gpa()
822 			ret = send_message_failure(cb, mqd, dsr, clines);  in gru_send_message_gpa()
824 	gru_free_cpu_resources(cb, dsr);  in gru_send_message_gpa()
905 	void *cb;  in gru_read_gpa()  local
910 	if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr))  in gru_read_gpa()
913 	gru_vload_phys(cb, gpa, gru_get_tri(dsr), iaa, IMA);  in gru_read_gpa()
914 	ret = gru_wait(cb);  in gru_read_gpa()
917 	gru_free_cpu_resources(cb, dsr);  in gru_read_gpa()
929 	void *cb;  in gru_copy_gpa()  local
934 	if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr))  in gru_copy_gpa()
936 	gru_bcopy(cb, src_gpa, dest_gpa, gru_get_tri(dsr),  in gru_copy_gpa()
938 	ret = gru_wait(cb);  in gru_copy_gpa()
939 	gru_free_cpu_resources(cb, dsr);  in gru_copy_gpa()
951 	void *cb;  in quicktest0()  local
956 	if (gru_get_cpu_resources(GRU_CACHE_LINE_BYTES, &cb, &dsr))  in quicktest0()
962 	gru_vload(cb, uv_gpa(&word0), gru_get_tri(dsr), XTYPE_DW, 1, 1, IMA);  in quicktest0()
963 	if (gru_wait(cb) != CBS_IDLE) {  in quicktest0()
972 	gru_vstore(cb, uv_gpa(&word1), gru_get_tri(dsr), XTYPE_DW, 1, 1, IMA);  in quicktest0()
973 	if (gru_wait(cb) != CBS_IDLE) {  in quicktest0()
987 	gru_free_cpu_resources(cb, dsr);  in quicktest0()
1048 	void *cb0, *cb;  in quicktest2()  local
1073 			cb = cb0 + i * GRU_HANDLE_STRIDE;  in quicktest2()
1074 			istatus = gru_check_status(cb);  in quicktest2()
1090 		gen = cb;  in quicktest2()