61 static __maybe_unused u64 fbnic_ts40_to_ns(struct fbnic_net *fbn, u64 ts40)  in fbnic_ts40_to_ns()  argument
70 		s = u64_stats_fetch_begin(&fbn->time_seq);  in fbnic_ts40_to_ns()
71 		offset = READ_ONCE(fbn->time_offset);  in fbnic_ts40_to_ns()
72 	} while (u64_stats_fetch_retry(&fbn->time_seq, s));  in fbnic_ts40_to_ns()
74 	high = READ_ONCE(fbn->time_high);  in fbnic_ts40_to_ns()
160 	struct fbnic_net *fbn;  in fbnic_tx_tstamp()  local
165 	fbn = netdev_priv(skb->dev);  in fbnic_tx_tstamp()
166 	if (fbn->hwtstamp_config.tx_type == HWTSTAMP_TX_OFF)  in fbnic_tx_tstamp()
353 	struct fbnic_net *fbn = netdev_priv(dev);  in fbnic_xmit_frame()  local
356 	return fbnic_xmit_frame_ring(skb, fbn->tx[q_map]);  in fbnic_xmit_frame()
469 	struct fbnic_net *fbn;  in fbnic_clean_tsq()  local
493 	fbn = netdev_priv(nv->napi.dev);  in fbnic_clean_tsq()
494 	ns = fbnic_ts40_to_ns(fbn, FIELD_GET(FBNIC_TCD_TYPE1_TS_MASK, tcd));  in fbnic_clean_tsq()
862 	struct fbnic_net *fbn;  in fbnic_rx_tstamp()  local
868 	fbn = netdev_priv(nv->napi.dev);  in fbnic_rx_tstamp()
870 	ns = fbnic_ts40_to_ns(fbn, ts);  in fbnic_rx_tstamp()
1048 void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn,  in fbnic_aggregate_ring_rx_counters()  argument
1054 	fbn->rx_stats.bytes += stats->bytes;  in fbnic_aggregate_ring_rx_counters()
1055 	fbn->rx_stats.packets += stats->packets;  in fbnic_aggregate_ring_rx_counters()
1056 	fbn->rx_stats.dropped += stats->dropped;  in fbnic_aggregate_ring_rx_counters()
1059 void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn,  in fbnic_aggregate_ring_tx_counters()  argument
1065 	fbn->tx_stats.bytes += stats->bytes;  in fbnic_aggregate_ring_tx_counters()
1066 	fbn->tx_stats.packets += stats->packets;  in fbnic_aggregate_ring_tx_counters()
1067 	fbn->tx_stats.dropped += stats->dropped;  in fbnic_aggregate_ring_tx_counters()
1068 	fbn->tx_stats.ts_lost += stats->ts_lost;  in fbnic_aggregate_ring_tx_counters()
1069 	fbn->tx_stats.ts_packets += stats->ts_packets;  in fbnic_aggregate_ring_tx_counters()
1072 static void fbnic_remove_tx_ring(struct fbnic_net *fbn,  in fbnic_remove_tx_ring()  argument
1078 	fbnic_aggregate_ring_tx_counters(fbn, txr);  in fbnic_remove_tx_ring()
1081 	WARN_ON(fbn->tx[txr->q_idx] && fbn->tx[txr->q_idx] != txr);  in fbnic_remove_tx_ring()
1082 	fbn->tx[txr->q_idx] = NULL;  in fbnic_remove_tx_ring()
1085 static void fbnic_remove_rx_ring(struct fbnic_net *fbn,  in fbnic_remove_rx_ring()  argument
1091 	fbnic_aggregate_ring_rx_counters(fbn, rxr);  in fbnic_remove_rx_ring()
1094 	WARN_ON(fbn->rx[rxr->q_idx] && fbn->rx[rxr->q_idx] != rxr);  in fbnic_remove_rx_ring()
1095 	fbn->rx[rxr->q_idx] = NULL;  in fbnic_remove_rx_ring()
1098 static void fbnic_free_napi_vector(struct fbnic_net *fbn,  in fbnic_free_napi_vector()  argument
1105 		fbnic_remove_tx_ring(fbn, &nv->qt[i].sub0);  in fbnic_free_napi_vector()
1106 		fbnic_remove_tx_ring(fbn, &nv->qt[i].cmpl);  in fbnic_free_napi_vector()
1110 		fbnic_remove_rx_ring(fbn, &nv->qt[i].sub0);  in fbnic_free_napi_vector()
1111 		fbnic_remove_rx_ring(fbn, &nv->qt[i].sub1);  in fbnic_free_napi_vector()
1112 		fbnic_remove_rx_ring(fbn, &nv->qt[i].cmpl);  in fbnic_free_napi_vector()
1118 	fbn->napi[fbnic_napi_idx(nv)] = NULL;  in fbnic_free_napi_vector()
1122 void fbnic_free_napi_vectors(struct fbnic_net *fbn)  in fbnic_free_napi_vectors()  argument
1126 	for (i = 0; i < fbn->num_napi; i++)  in fbnic_free_napi_vectors()
1127 		if (fbn->napi[i])  in fbnic_free_napi_vectors()
1128 			fbnic_free_napi_vector(fbn, fbn->napi[i]);  in fbnic_free_napi_vectors()
1134 static int fbnic_alloc_nv_page_pool(struct fbnic_net *fbn,  in fbnic_alloc_nv_page_pool()  argument
1140 		.pool_size = (fbn->hpq_size + fbn->ppq_size) * nv->rxt_count,  in fbnic_alloc_nv_page_pool()
1179 static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn,  in fbnic_alloc_napi_vector()  argument
1213 	fbn->napi[fbnic_napi_idx(nv)] = nv;  in fbnic_alloc_napi_vector()
1214 	netif_napi_add(fbn->netdev, &nv->napi, fbnic_poll);  in fbnic_alloc_napi_vector()
1225 		err = fbnic_alloc_nv_page_pool(fbn, nv);  in fbnic_alloc_napi_vector()
1247 			fbn->tx[txq_idx] = &qt->sub0;  in fbnic_alloc_napi_vector()
1278 		fbn->rx[rxq_idx] = &qt->cmpl;  in fbnic_alloc_napi_vector()
1294 	fbn->napi[fbnic_napi_idx(nv)] = NULL;  in fbnic_alloc_napi_vector()
1299 int fbnic_alloc_napi_vectors(struct fbnic_net *fbn)  in fbnic_alloc_napi_vectors()  argument
1302 	unsigned int num_tx = fbn->num_tx_queues;  in fbnic_alloc_napi_vectors()
1303 	unsigned int num_rx = fbn->num_rx_queues;  in fbnic_alloc_napi_vectors()
1304 	unsigned int num_napi = fbn->num_napi;  in fbnic_alloc_napi_vectors()
1305 	struct fbnic_dev *fbd = fbn->fbd;  in fbnic_alloc_napi_vectors()
1311 			err = fbnic_alloc_napi_vector(fbd, fbn,  in fbnic_alloc_napi_vectors()
1330 		err = fbnic_alloc_napi_vector(fbd, fbn, num_napi, v_idx,  in fbnic_alloc_napi_vectors()
1348 	fbnic_free_napi_vectors(fbn);  in fbnic_alloc_napi_vectors()
1368 static int fbnic_alloc_tx_ring_desc(struct fbnic_net *fbn,  in fbnic_alloc_tx_ring_desc()  argument
1371 	struct device *dev = fbn->netdev->dev.parent;  in fbnic_alloc_tx_ring_desc()
1375 	size = ALIGN(array_size(sizeof(*txr->desc), fbn->txq_size), 4096);  in fbnic_alloc_tx_ring_desc()
1383 	txr->size_mask = fbn->txq_size - 1;  in fbnic_alloc_tx_ring_desc()
1398 static int fbnic_alloc_tx_ring_resources(struct fbnic_net *fbn,  in fbnic_alloc_tx_ring_resources()  argument
1401 	struct device *dev = fbn->netdev->dev.parent;  in fbnic_alloc_tx_ring_resources()
1407 	err = fbnic_alloc_tx_ring_desc(fbn, txr);  in fbnic_alloc_tx_ring_resources()
1425 static int fbnic_alloc_rx_ring_desc(struct fbnic_net *fbn,  in fbnic_alloc_rx_ring_desc()  argument
1428 	struct device *dev = fbn->netdev->dev.parent;  in fbnic_alloc_rx_ring_desc()
1435 		rxq_size = fbn->hpq_size / FBNIC_BD_FRAG_COUNT;  in fbnic_alloc_rx_ring_desc()
1439 		rxq_size = fbn->ppq_size / FBNIC_BD_FRAG_COUNT;  in fbnic_alloc_rx_ring_desc()
1443 		rxq_size = fbn->rcq_size;  in fbnic_alloc_rx_ring_desc()
1478 static int fbnic_alloc_rx_ring_resources(struct fbnic_net *fbn,  in fbnic_alloc_rx_ring_resources()  argument
1481 	struct device *dev = fbn->netdev->dev.parent;  in fbnic_alloc_rx_ring_resources()
1484 	err = fbnic_alloc_rx_ring_desc(fbn, rxr);  in fbnic_alloc_rx_ring_resources()
1499 static void fbnic_free_qt_resources(struct fbnic_net *fbn,  in fbnic_free_qt_resources()  argument
1502 	struct device *dev = fbn->netdev->dev.parent;  in fbnic_free_qt_resources()
1509 static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn,  in fbnic_alloc_tx_qt_resources()  argument
1512 	struct device *dev = fbn->netdev->dev.parent;  in fbnic_alloc_tx_qt_resources()
1515 	err = fbnic_alloc_tx_ring_resources(fbn, &qt->sub0);  in fbnic_alloc_tx_qt_resources()
1519 	err = fbnic_alloc_tx_ring_resources(fbn, &qt->cmpl);  in fbnic_alloc_tx_qt_resources()
1530 static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn,  in fbnic_alloc_rx_qt_resources()  argument
1533 	struct device *dev = fbn->netdev->dev.parent;  in fbnic_alloc_rx_qt_resources()
1536 	err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0);  in fbnic_alloc_rx_qt_resources()
1540 	err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub1);  in fbnic_alloc_rx_qt_resources()
1544 	err = fbnic_alloc_rx_ring_resources(fbn, &qt->cmpl);  in fbnic_alloc_rx_qt_resources()
1557 static void fbnic_free_nv_resources(struct fbnic_net *fbn,  in fbnic_free_nv_resources()  argument
1564 		fbnic_free_qt_resources(fbn, &nv->qt[i]);  in fbnic_free_nv_resources()
1567 		fbnic_free_qt_resources(fbn, &nv->qt[i]);  in fbnic_free_nv_resources()
1570 static int fbnic_alloc_nv_resources(struct fbnic_net *fbn,  in fbnic_alloc_nv_resources()  argument
1577 		err = fbnic_alloc_tx_qt_resources(fbn, &nv->qt[i]);  in fbnic_alloc_nv_resources()
1584 		err = fbnic_alloc_rx_qt_resources(fbn, &nv->qt[i]);  in fbnic_alloc_nv_resources()
1593 		fbnic_free_qt_resources(fbn, &nv->qt[i]);  in fbnic_alloc_nv_resources()
1597 void fbnic_free_resources(struct fbnic_net *fbn)  in fbnic_free_resources()  argument
1601 	for (i = 0; i < fbn->num_napi; i++)  in fbnic_free_resources()
1602 		fbnic_free_nv_resources(fbn, fbn->napi[i]);  in fbnic_free_resources()
1605 int fbnic_alloc_resources(struct fbnic_net *fbn)  in fbnic_alloc_resources()  argument
1609 	for (i = 0; i < fbn->num_napi; i++) {  in fbnic_alloc_resources()
1610 		err = fbnic_alloc_nv_resources(fbn, fbn->napi[i]);  in fbnic_alloc_resources()
1619 		fbnic_free_nv_resources(fbn, fbn->napi[i]);  in fbnic_alloc_resources()
1666 int fbnic_set_netif_queues(struct fbnic_net *fbn)  in fbnic_set_netif_queues()  argument
1670 	err = netif_set_real_num_queues(fbn->netdev, fbn->num_tx_queues,  in fbnic_set_netif_queues()
1671 					fbn->num_rx_queues);  in fbnic_set_netif_queues()
1675 	for (i = 0; i < fbn->num_napi; i++)  in fbnic_set_netif_queues()
1676 		fbnic_set_netif_napi(fbn->napi[i]);  in fbnic_set_netif_queues()
1681 void fbnic_reset_netif_queues(struct fbnic_net *fbn)  in fbnic_reset_netif_queues()  argument
1685 	for (i = 0; i < fbn->num_napi; i++)  in fbnic_reset_netif_queues()
1686 		fbnic_reset_netif_napi(fbn->napi[i]);  in fbnic_reset_netif_queues()
1719 void fbnic_napi_disable(struct fbnic_net *fbn)  in fbnic_napi_disable()  argument
1723 	for (i = 0; i < fbn->num_napi; i++) {  in fbnic_napi_disable()
1724 		napi_disable(&fbn->napi[i]->napi);  in fbnic_napi_disable()
1726 		fbnic_nv_irq_disable(fbn->napi[i]);  in fbnic_napi_disable()
1730 void fbnic_disable(struct fbnic_net *fbn)  in fbnic_disable()  argument
1732 	struct fbnic_dev *fbd = fbn->fbd;  in fbnic_disable()
1735 	for (i = 0; i < fbn->num_napi; i++) {  in fbnic_disable()
1736 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_disable()
1842 void fbnic_flush(struct fbnic_net *fbn)  in fbnic_flush()  argument
1846 	for (i = 0; i < fbn->num_napi; i++) {  in fbnic_flush()
1847 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_flush()
1888 void fbnic_fill(struct fbnic_net *fbn)  in fbnic_fill()  argument
1892 	for (i = 0; i < fbn->num_napi; i++) {  in fbnic_fill()
1893 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_fill()
2051 void fbnic_enable(struct fbnic_net *fbn)  in fbnic_enable()  argument
2053 	struct fbnic_dev *fbd = fbn->fbd;  in fbnic_enable()
2056 	for (i = 0; i < fbn->num_napi; i++) {  in fbnic_enable()
2057 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_enable()
2091 void fbnic_napi_enable(struct fbnic_net *fbn)  in fbnic_napi_enable()  argument
2094 	struct fbnic_dev *fbd = fbn->fbd;  in fbnic_napi_enable()
2097 	for (i = 0; i < fbn->num_napi; i++) {  in fbnic_napi_enable()
2098 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_napi_enable()
2125 	struct fbnic_net *fbn = netdev_priv(netdev);  in fbnic_napi_depletion_check()  local
2127 	struct fbnic_dev *fbd = fbn->fbd;  in fbnic_napi_depletion_check()
2130 	for (i = 0; i < fbn->num_napi; i++) {  in fbnic_napi_depletion_check()
2131 		struct fbnic_napi_vector *nv = fbn->napi[i];  in fbnic_napi_depletion_check()