/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2022 Intel Corporation */ #include #include #include #include #include #include #include #include "idpf_ethdev.h" #include "idpf_rxtx.h" #define IDPF_TX_SINGLE_Q "tx_single" #define IDPF_RX_SINGLE_Q "rx_single" #define IDPF_VPORT "vport" rte_spinlock_t idpf_adapter_lock; /* A list for all adapters, one adapter matches one PCI device */ struct idpf_adapter_list idpf_adapter_list; bool idpf_adapter_list_init; uint64_t idpf_timestamp_dynflag; static const char * const idpf_valid_args[] = { IDPF_TX_SINGLE_Q, IDPF_RX_SINGLE_Q, IDPF_VPORT, NULL }; static int idpf_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) { struct rte_eth_link new_link; memset(&new_link, 0, sizeof(new_link)); new_link.link_speed = RTE_ETH_SPEED_NUM_NONE; new_link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; new_link.link_autoneg = !(dev->data->dev_conf.link_speeds & RTE_ETH_LINK_SPEED_FIXED); return rte_eth_linkstatus_set(dev, &new_link); } static int idpf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct idpf_vport *vport = dev->data->dev_private; struct idpf_adapter *adapter = vport->adapter; dev_info->max_rx_queues = adapter->caps->max_rx_q; dev_info->max_tx_queues = adapter->caps->max_tx_q; dev_info->min_rx_bufsize = IDPF_MIN_BUF_SIZE; dev_info->max_rx_pktlen = IDPF_MAX_FRAME_SIZE; dev_info->max_mtu = dev_info->max_rx_pktlen - IDPF_ETH_OVERHEAD; dev_info->min_mtu = RTE_ETHER_MIN_MTU; dev_info->flow_type_rss_offloads = IDPF_RSS_OFFLOAD_ALL; dev_info->rx_offload_capa = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | RTE_ETH_RX_OFFLOAD_UDP_CKSUM | RTE_ETH_RX_OFFLOAD_TCP_CKSUM | RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM | RTE_ETH_RX_OFFLOAD_TIMESTAMP; dev_info->tx_offload_capa = RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | RTE_ETH_TX_OFFLOAD_UDP_CKSUM | RTE_ETH_TX_OFFLOAD_TCP_CKSUM | RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_MULTI_SEGS | RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; dev_info->default_txconf = (struct rte_eth_txconf) { .tx_free_thresh = IDPF_DEFAULT_TX_FREE_THRESH, .tx_rs_thresh = IDPF_DEFAULT_TX_RS_THRESH, }; dev_info->default_rxconf = (struct rte_eth_rxconf) { .rx_free_thresh = IDPF_DEFAULT_RX_FREE_THRESH, }; dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { .nb_max = IDPF_MAX_RING_DESC, .nb_min = IDPF_MIN_RING_DESC, .nb_align = IDPF_ALIGN_RING_DESC, }; dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { .nb_max = IDPF_MAX_RING_DESC, .nb_min = IDPF_MIN_RING_DESC, .nb_align = IDPF_ALIGN_RING_DESC, }; return 0; } static int idpf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu __rte_unused) { /* mtu setting is forbidden if port is start */ if (dev->data->dev_started) { PMD_DRV_LOG(ERR, "port must be stopped before configuration"); return -EBUSY; } return 0; } static const uint32_t * idpf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) { static const uint32_t ptypes[] = { RTE_PTYPE_L2_ETHER, RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, RTE_PTYPE_L4_FRAG, RTE_PTYPE_L4_UDP, RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_SCTP, RTE_PTYPE_L4_ICMP, RTE_PTYPE_UNKNOWN }; return ptypes; } static int idpf_init_vport_req_info(struct rte_eth_dev *dev) { struct idpf_vport *vport = dev->data->dev_private; struct idpf_adapter *adapter = vport->adapter; struct virtchnl2_create_vport *vport_info; uint16_t idx = adapter->cur_vport_idx; if (idx == IDPF_INVALID_VPORT_IDX) { PMD_INIT_LOG(ERR, "Invalid vport index."); return -EINVAL; } if (adapter->vport_req_info[idx] == NULL) { adapter->vport_req_info[idx] = rte_zmalloc(NULL, sizeof(struct virtchnl2_create_vport), 0); if (adapter->vport_req_info[idx] == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate vport_req_info"); return -ENOMEM; } } vport_info = (struct virtchnl2_create_vport *)adapter->vport_req_info[idx]; vport_info->vport_type = rte_cpu_to_le_16(VIRTCHNL2_VPORT_TYPE_DEFAULT); if (adapter->txq_model == 0) { vport_info->txq_model = rte_cpu_to_le_16(VIRTCHNL2_QUEUE_MODEL_SPLIT); vport_info->num_tx_q = IDPF_DEFAULT_TXQ_NUM; vport_info->num_tx_complq = IDPF_DEFAULT_TXQ_NUM * IDPF_TX_COMPLQ_PER_GRP; } else { vport_info->txq_model = rte_cpu_to_le_16(VIRTCHNL2_QUEUE_MODEL_SINGLE); vport_info->num_tx_q = IDPF_DEFAULT_TXQ_NUM; vport_info->num_tx_complq = 0; } if (adapter->rxq_model == 0) { vport_info->rxq_model = rte_cpu_to_le_16(VIRTCHNL2_QUEUE_MODEL_SPLIT); vport_info->num_rx_q = IDPF_DEFAULT_RXQ_NUM; vport_info->num_rx_bufq = IDPF_DEFAULT_RXQ_NUM * IDPF_RX_BUFQ_PER_GRP; } else { vport_info->rxq_model = rte_cpu_to_le_16(VIRTCHNL2_QUEUE_MODEL_SINGLE); vport_info->num_rx_q = IDPF_DEFAULT_RXQ_NUM; vport_info->num_rx_bufq = 0; } return 0; } static int idpf_parse_devarg_id(char *name) { uint16_t val; char *p; p = strstr(name, "vport_"); if (p == NULL) return -EINVAL; p += sizeof("vport_") - 1; val = strtoul(p, NULL, 10); return val; } #define IDPF_RSS_KEY_LEN 52 static int idpf_init_vport(struct rte_eth_dev *dev) { struct idpf_vport *vport = dev->data->dev_private; struct idpf_adapter *adapter = vport->adapter; uint16_t idx = adapter->cur_vport_idx; struct virtchnl2_create_vport *vport_info = (struct virtchnl2_create_vport *)adapter->vport_recv_info[idx]; int i, type, ret; vport->vport_id = vport_info->vport_id; vport->txq_model = vport_info->txq_model; vport->rxq_model = vport_info->rxq_model; vport->num_tx_q = vport_info->num_tx_q; vport->num_tx_complq = vport_info->num_tx_complq; vport->num_rx_q = vport_info->num_rx_q; vport->num_rx_bufq = vport_info->num_rx_bufq; vport->max_mtu = vport_info->max_mtu; rte_memcpy(vport->default_mac_addr, vport_info->default_mac_addr, ETH_ALEN); vport->rss_algorithm = vport_info->rss_algorithm; vport->rss_key_size = RTE_MIN(IDPF_RSS_KEY_LEN, vport_info->rss_key_size); vport->rss_lut_size = vport_info->rss_lut_size; vport->sw_idx = idx; for (i = 0; i < vport_info->chunks.num_chunks; i++) { type = vport_info->chunks.chunks[i].type; switch (type) { case VIRTCHNL2_QUEUE_TYPE_TX: vport->chunks_info.tx_start_qid = vport_info->chunks.chunks[i].start_queue_id; vport->chunks_info.tx_qtail_start = vport_info->chunks.chunks[i].qtail_reg_start; vport->chunks_info.tx_qtail_spacing = vport_info->chunks.chunks[i].qtail_reg_spacing; break; case VIRTCHNL2_QUEUE_TYPE_RX: vport->chunks_info.rx_start_qid = vport_info->chunks.chunks[i].start_queue_id; vport->chunks_info.rx_qtail_start = vport_info->chunks.chunks[i].qtail_reg_start; vport->chunks_info.rx_qtail_spacing = vport_info->chunks.chunks[i].qtail_reg_spacing; break; case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: vport->chunks_info.tx_compl_start_qid = vport_info->chunks.chunks[i].start_queue_id; vport->chunks_info.tx_compl_qtail_start = vport_info->chunks.chunks[i].qtail_reg_start; vport->chunks_info.tx_compl_qtail_spacing = vport_info->chunks.chunks[i].qtail_reg_spacing; break; case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: vport->chunks_info.rx_buf_start_qid = vport_info->chunks.chunks[i].start_queue_id; vport->chunks_info.rx_buf_qtail_start = vport_info->chunks.chunks[i].qtail_reg_start; vport->chunks_info.rx_buf_qtail_spacing = vport_info->chunks.chunks[i].qtail_reg_spacing; break; default: PMD_INIT_LOG(ERR, "Unsupported queue type"); break; } } ret = idpf_parse_devarg_id(dev->data->name); if (ret < 0) { PMD_INIT_LOG(ERR, "Failed to parse devarg id."); return -EINVAL; } vport->devarg_id = ret; vport->dev_data = dev->data; adapter->vports[idx] = vport; return 0; } static int idpf_config_rss(struct idpf_vport *vport) { int ret; ret = idpf_vc_set_rss_key(vport); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to configure RSS key"); return ret; } ret = idpf_vc_set_rss_lut(vport); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to configure RSS lut"); return ret; } ret = idpf_vc_set_rss_hash(vport); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to configure RSS hash"); return ret; } return ret; } static int idpf_init_rss(struct idpf_vport *vport) { struct rte_eth_rss_conf *rss_conf; uint16_t i, nb_q, lut_size; int ret = 0; rss_conf = &vport->dev_data->dev_conf.rx_adv_conf.rss_conf; nb_q = vport->dev_data->nb_rx_queues; vport->rss_key = rte_zmalloc("rss_key", vport->rss_key_size, 0); if (vport->rss_key == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate RSS key"); ret = -ENOMEM; goto err_alloc_key; } lut_size = vport->rss_lut_size; vport->rss_lut = rte_zmalloc("rss_lut", sizeof(uint32_t) * lut_size, 0); if (vport->rss_lut == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate RSS lut"); ret = -ENOMEM; goto err_alloc_lut; } if (rss_conf->rss_key == NULL) { for (i = 0; i < vport->rss_key_size; i++) vport->rss_key[i] = (uint8_t)rte_rand(); } else if (rss_conf->rss_key_len != vport->rss_key_size) { PMD_INIT_LOG(ERR, "Invalid RSS key length in RSS configuration, should be %d", vport->rss_key_size); ret = -EINVAL; goto err_cfg_key; } else { rte_memcpy(vport->rss_key, rss_conf->rss_key, vport->rss_key_size); } for (i = 0; i < lut_size; i++) vport->rss_lut[i] = i % nb_q; vport->rss_hf = IDPF_DEFAULT_RSS_HASH_EXPANDED; ret = idpf_config_rss(vport); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to configure RSS"); goto err_cfg_key; } return ret; err_cfg_key: rte_free(vport->rss_lut); vport->rss_lut = NULL; err_alloc_lut: rte_free(vport->rss_key); vport->rss_key = NULL; err_alloc_key: return ret; } static int idpf_dev_configure(struct rte_eth_dev *dev) { struct idpf_vport *vport = dev->data->dev_private; struct rte_eth_conf *conf = &dev->data->dev_conf; struct idpf_adapter *adapter = vport->adapter; int ret; if (conf->link_speeds & RTE_ETH_LINK_SPEED_FIXED) { PMD_INIT_LOG(ERR, "Setting link speed is not supported"); return -ENOTSUP; } if (conf->txmode.mq_mode != RTE_ETH_MQ_TX_NONE) { PMD_INIT_LOG(ERR, "Multi-queue TX mode %d is not supported", conf->txmode.mq_mode); return -ENOTSUP; } if (conf->lpbk_mode != 0) { PMD_INIT_LOG(ERR, "Loopback operation mode %d is not supported", conf->lpbk_mode); return -ENOTSUP; } if (conf->dcb_capability_en != 0) { PMD_INIT_LOG(ERR, "Priority Flow Control(PFC) if not supported"); return -ENOTSUP; } if (conf->intr_conf.lsc != 0) { PMD_INIT_LOG(ERR, "LSC interrupt is not supported"); return -ENOTSUP; } if (conf->intr_conf.rxq != 0) { PMD_INIT_LOG(ERR, "RXQ interrupt is not supported"); return -ENOTSUP; } if (conf->intr_conf.rmv != 0) { PMD_INIT_LOG(ERR, "RMV interrupt is not supported"); return -ENOTSUP; } if (adapter->caps->rss_caps != 0 && dev->data->nb_rx_queues != 0) { ret = idpf_init_rss(vport); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to init rss"); return ret; } } else { PMD_INIT_LOG(ERR, "RSS is not supported."); return -1; } return 0; } static int idpf_config_rx_queues_irqs(struct rte_eth_dev *dev) { struct idpf_vport *vport = dev->data->dev_private; struct idpf_adapter *adapter = vport->adapter; struct virtchnl2_queue_vector *qv_map; struct idpf_hw *hw = &adapter->hw; uint32_t dynctl_reg_start; uint32_t itrn_reg_start; uint32_t dynctl_val, itrn_val; uint16_t i; qv_map = rte_zmalloc("qv_map", dev->data->nb_rx_queues * sizeof(struct virtchnl2_queue_vector), 0); if (qv_map == NULL) { PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map", dev->data->nb_rx_queues); goto qv_map_alloc_err; } /* Rx interrupt disabled, Map interrupt only for writeback */ /* The capability flags adapter->caps->other_caps should be * compared with bit VIRTCHNL2_CAP_WB_ON_ITR here. The if * condition should be updated when the FW can return the * correct flag bits. */ dynctl_reg_start = vport->recv_vectors->vchunks.vchunks->dynctl_reg_start; itrn_reg_start = vport->recv_vectors->vchunks.vchunks->itrn_reg_start; dynctl_val = IDPF_READ_REG(hw, dynctl_reg_start); PMD_DRV_LOG(DEBUG, "Value of dynctl_reg_start is 0x%x", dynctl_val); itrn_val = IDPF_READ_REG(hw, itrn_reg_start); PMD_DRV_LOG(DEBUG, "Value of itrn_reg_start is 0x%x", itrn_val); /* Force write-backs by setting WB_ON_ITR bit in DYN_CTL * register. WB_ON_ITR and INTENA are mutually exclusive * bits. Setting WB_ON_ITR bits means TX and RX Descs * are written back based on ITR expiration irrespective * of INTENA setting. */ /* TBD: need to tune INTERVAL value for better performance. */ if (itrn_val != 0) IDPF_WRITE_REG(hw, dynctl_reg_start, VIRTCHNL2_ITR_IDX_0 << PF_GLINT_DYN_CTL_ITR_INDX_S | PF_GLINT_DYN_CTL_WB_ON_ITR_M | itrn_val << PF_GLINT_DYN_CTL_INTERVAL_S); else IDPF_WRITE_REG(hw, dynctl_reg_start, VIRTCHNL2_ITR_IDX_0 << PF_GLINT_DYN_CTL_ITR_INDX_S | PF_GLINT_DYN_CTL_WB_ON_ITR_M | IDPF_DFLT_INTERVAL << PF_GLINT_DYN_CTL_INTERVAL_S); for (i = 0; i < dev->data->nb_rx_queues; i++) { /* map all queues to the same vector */ qv_map[i].queue_id = vport->chunks_info.rx_start_qid + i; qv_map[i].vector_id = vport->recv_vectors->vchunks.vchunks->start_vector_id; } vport->qv_map = qv_map; if (idpf_vc_config_irq_map_unmap(vport, true) != 0) { PMD_DRV_LOG(ERR, "config interrupt mapping failed"); goto config_irq_map_err; } return 0; config_irq_map_err: rte_free(vport->qv_map); vport->qv_map = NULL; qv_map_alloc_err: return -1; } static int idpf_start_queues(struct rte_eth_dev *dev) { struct idpf_rx_queue *rxq; struct idpf_tx_queue *txq; int err = 0; int i; for (i = 0; i < dev->data->nb_tx_queues; i++) { txq = dev->data->tx_queues[i]; if (txq == NULL || txq->tx_deferred_start) continue; err = idpf_tx_queue_start(dev, i); if (err != 0) { PMD_DRV_LOG(ERR, "Fail to start Tx queue %u", i); return err; } } for (i = 0; i < dev->data->nb_rx_queues; i++) { rxq = dev->data->rx_queues[i]; if (rxq == NULL || rxq->rx_deferred_start) continue; err = idpf_rx_queue_start(dev, i); if (err != 0) { PMD_DRV_LOG(ERR, "Fail to start Rx queue %u", i); return err; } } return err; } static int idpf_dev_start(struct rte_eth_dev *dev) { struct idpf_vport *vport = dev->data->dev_private; struct idpf_adapter *adapter = vport->adapter; uint16_t num_allocated_vectors = adapter->caps->num_allocated_vectors; uint16_t req_vecs_num; int ret; if (dev->data->mtu > vport->max_mtu) { PMD_DRV_LOG(ERR, "MTU should be less than %d", vport->max_mtu); ret = -EINVAL; goto err_mtu; } vport->max_pkt_len = dev->data->mtu + IDPF_ETH_OVERHEAD; req_vecs_num = IDPF_DFLT_Q_VEC_NUM; if (req_vecs_num + adapter->used_vecs_num > num_allocated_vectors) { PMD_DRV_LOG(ERR, "The accumulated request vectors' number should be less than %d", num_allocated_vectors); ret = -EINVAL; goto err_mtu; } ret = idpf_vc_alloc_vectors(vport, req_vecs_num); if (ret != 0) { PMD_DRV_LOG(ERR, "Failed to allocate interrupt vectors"); goto err_mtu; } adapter->used_vecs_num += req_vecs_num; ret = idpf_config_rx_queues_irqs(dev); if (ret != 0) { PMD_DRV_LOG(ERR, "Failed to configure irqs"); goto err_mtu; } ret = idpf_start_queues(dev); if (ret != 0) { PMD_DRV_LOG(ERR, "Failed to start queues"); goto err_mtu; } idpf_set_rx_function(dev); idpf_set_tx_function(dev); ret = idpf_vc_ena_dis_vport(vport, true); if (ret != 0) { PMD_DRV_LOG(ERR, "Failed to enable vport"); goto err_vport; } return 0; err_vport: idpf_stop_queues(dev); err_mtu: return ret; } static int idpf_dev_stop(struct rte_eth_dev *dev) { struct idpf_vport *vport = dev->data->dev_private; if (dev->data->dev_started == 0) return 0; idpf_vc_ena_dis_vport(vport, false); idpf_stop_queues(dev); idpf_vc_config_irq_map_unmap(vport, false); if (vport->recv_vectors != NULL) idpf_vc_dealloc_vectors(vport); return 0; } static int idpf_dev_close(struct rte_eth_dev *dev) { struct idpf_vport *vport = dev->data->dev_private; struct idpf_adapter *adapter = vport->adapter; idpf_dev_stop(dev); idpf_vc_destroy_vport(vport); rte_free(vport->rss_lut); vport->rss_lut = NULL; rte_free(vport->rss_key); vport->rss_key = NULL; rte_free(vport->recv_vectors); vport->recv_vectors = NULL; rte_free(vport->qv_map); vport->qv_map = NULL; adapter->cur_vports &= ~RTE_BIT32(vport->devarg_id); rte_free(vport); dev->data->dev_private = NULL; return 0; } static int insert_value(struct idpf_adapter *adapter, uint16_t id) { uint16_t i; for (i = 0; i < adapter->req_vport_nb; i++) { if (adapter->req_vports[i] == id) return 0; } if (adapter->req_vport_nb >= RTE_DIM(adapter->req_vports)) { PMD_INIT_LOG(ERR, "Total vport number can't be > %d", IDPF_MAX_VPORT_NUM); return -EINVAL; } adapter->req_vports[adapter->req_vport_nb] = id; adapter->req_vport_nb++; return 0; } static const char * parse_range(const char *value, struct idpf_adapter *adapter) { uint16_t lo, hi, i; int n = 0; int result; const char *pos = value; result = sscanf(value, "%hu%n-%hu%n", &lo, &n, &hi, &n); if (result == 1) { if (lo >= IDPF_MAX_VPORT_NUM) return NULL; if (insert_value(adapter, lo) != 0) return NULL; } else if (result == 2) { if (lo > hi || hi >= IDPF_MAX_VPORT_NUM) return NULL; for (i = lo; i <= hi; i++) { if (insert_value(adapter, i) != 0) return NULL; } } else { return NULL; } return pos + n; } static int parse_vport(const char *key, const char *value, void *args) { struct idpf_adapter *adapter = args; const char *pos = value; int i; adapter->req_vport_nb = 0; if (*pos == '[') pos++; while (1) { pos = parse_range(pos, adapter); if (pos == NULL) { PMD_INIT_LOG(ERR, "invalid value:\"%s\" for key:\"%s\", ", value, key); return -EINVAL; } if (*pos != ',') break; pos++; } if (*value == '[' && *pos != ']') { PMD_INIT_LOG(ERR, "invalid value:\"%s\" for key:\"%s\", ", value, key); return -EINVAL; } if (adapter->cur_vport_nb + adapter->req_vport_nb > IDPF_MAX_VPORT_NUM) { PMD_INIT_LOG(ERR, "Total vport number can't be > %d", IDPF_MAX_VPORT_NUM); return -EINVAL; } for (i = 0; i < adapter->req_vport_nb; i++) { if ((adapter->cur_vports & RTE_BIT32(adapter->req_vports[i])) == 0) { adapter->cur_vports |= RTE_BIT32(adapter->req_vports[i]); adapter->cur_vport_nb++; } else { PMD_INIT_LOG(ERR, "Vport %d has been created", adapter->req_vports[i]); return -EINVAL; } } return 0; } static int parse_bool(const char *key, const char *value, void *args) { int *i = args; char *end; int num; errno = 0; num = strtoul(value, &end, 10); if (errno == ERANGE || (num != 0 && num != 1)) { PMD_INIT_LOG(ERR, "invalid value:\"%s\" for key:\"%s\", value must be 0 or 1", value, key); return -EINVAL; } *i = num; return 0; } static int idpf_parse_devargs(struct rte_pci_device *pci_dev, struct idpf_adapter *adapter) { struct rte_devargs *devargs = pci_dev->device.devargs; struct rte_kvargs *kvlist; int ret; if (devargs == NULL) return 0; kvlist = rte_kvargs_parse(devargs->args, idpf_valid_args); if (kvlist == NULL) { PMD_INIT_LOG(ERR, "invalid kvargs key"); return -EINVAL; } ret = rte_kvargs_process(kvlist, IDPF_VPORT, &parse_vport, adapter); if (ret != 0) goto bail; ret = rte_kvargs_process(kvlist, IDPF_TX_SINGLE_Q, &parse_bool, &adapter->txq_model); if (ret != 0) goto bail; ret = rte_kvargs_process(kvlist, IDPF_RX_SINGLE_Q, &parse_bool, &adapter->rxq_model); if (ret != 0) goto bail; bail: rte_kvargs_free(kvlist); return ret; } static void idpf_reset_pf(struct idpf_hw *hw) { uint32_t reg; reg = IDPF_READ_REG(hw, PFGEN_CTRL); IDPF_WRITE_REG(hw, PFGEN_CTRL, (reg | PFGEN_CTRL_PFSWR)); } #define IDPF_RESET_WAIT_CNT 100 static int idpf_check_pf_reset_done(struct idpf_hw *hw) { uint32_t reg; int i; for (i = 0; i < IDPF_RESET_WAIT_CNT; i++) { reg = IDPF_READ_REG(hw, PFGEN_RSTAT); if (reg != 0xFFFFFFFF && (reg & PFGEN_RSTAT_PFR_STATE_M)) return 0; rte_delay_ms(1000); } PMD_INIT_LOG(ERR, "IDPF reset timeout"); return -EBUSY; } #define CTLQ_NUM 2 static int idpf_init_mbx(struct idpf_hw *hw) { struct idpf_ctlq_create_info ctlq_info[CTLQ_NUM] = { { .type = IDPF_CTLQ_TYPE_MAILBOX_TX, .id = IDPF_CTLQ_ID, .len = IDPF_CTLQ_LEN, .buf_size = IDPF_DFLT_MBX_BUF_SIZE, .reg = { .head = PF_FW_ATQH, .tail = PF_FW_ATQT, .len = PF_FW_ATQLEN, .bah = PF_FW_ATQBAH, .bal = PF_FW_ATQBAL, .len_mask = PF_FW_ATQLEN_ATQLEN_M, .len_ena_mask = PF_FW_ATQLEN_ATQENABLE_M, .head_mask = PF_FW_ATQH_ATQH_M, } }, { .type = IDPF_CTLQ_TYPE_MAILBOX_RX, .id = IDPF_CTLQ_ID, .len = IDPF_CTLQ_LEN, .buf_size = IDPF_DFLT_MBX_BUF_SIZE, .reg = { .head = PF_FW_ARQH, .tail = PF_FW_ARQT, .len = PF_FW_ARQLEN, .bah = PF_FW_ARQBAH, .bal = PF_FW_ARQBAL, .len_mask = PF_FW_ARQLEN_ARQLEN_M, .len_ena_mask = PF_FW_ARQLEN_ARQENABLE_M, .head_mask = PF_FW_ARQH_ARQH_M, } } }; struct idpf_ctlq_info *ctlq; int ret; ret = idpf_ctlq_init(hw, CTLQ_NUM, ctlq_info); if (ret != 0) return ret; LIST_FOR_EACH_ENTRY_SAFE(ctlq, NULL, &hw->cq_list_head, struct idpf_ctlq_info, cq_list) { if (ctlq->q_id == IDPF_CTLQ_ID && ctlq->cq_type == IDPF_CTLQ_TYPE_MAILBOX_TX) hw->asq = ctlq; if (ctlq->q_id == IDPF_CTLQ_ID && ctlq->cq_type == IDPF_CTLQ_TYPE_MAILBOX_RX) hw->arq = ctlq; } if (hw->asq == NULL || hw->arq == NULL) { idpf_ctlq_deinit(hw); ret = -ENOENT; } return ret; } static int idpf_adapter_init(struct rte_pci_device *pci_dev, struct idpf_adapter *adapter) { struct idpf_hw *hw = &adapter->hw; int ret = 0; hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; hw->hw_addr_len = pci_dev->mem_resource[0].len; hw->back = adapter; hw->vendor_id = pci_dev->id.vendor_id; hw->device_id = pci_dev->id.device_id; hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; strncpy(adapter->name, pci_dev->device.name, PCI_PRI_STR_SIZE); idpf_reset_pf(hw); ret = idpf_check_pf_reset_done(hw); if (ret != 0) { PMD_INIT_LOG(ERR, "IDPF is still resetting"); goto err; } ret = idpf_init_mbx(hw); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to init mailbox"); goto err; } adapter->mbx_resp = rte_zmalloc("idpf_adapter_mbx_resp", IDPF_DFLT_MBX_BUF_SIZE, 0); if (adapter->mbx_resp == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate idpf_adapter_mbx_resp memory"); ret = -ENOMEM; goto err_mbx; } ret = idpf_vc_check_api_version(adapter); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to check api version"); goto err_api; } ret = idpf_get_pkt_type(adapter); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to set ptype table"); goto err_api; } adapter->caps = rte_zmalloc("idpf_caps", sizeof(struct virtchnl2_get_capabilities), 0); if (adapter->caps == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate idpf_caps memory"); ret = -ENOMEM; goto err_api; } ret = idpf_vc_get_caps(adapter); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to get capabilities"); goto err_caps; } adapter->max_vport_nb = adapter->caps->max_vports; adapter->vport_req_info = rte_zmalloc("vport_req_info", adapter->max_vport_nb * sizeof(*adapter->vport_req_info), 0); if (adapter->vport_req_info == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate vport_req_info memory"); ret = -ENOMEM; goto err_caps; } adapter->vport_recv_info = rte_zmalloc("vport_recv_info", adapter->max_vport_nb * sizeof(*adapter->vport_recv_info), 0); if (adapter->vport_recv_info == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate vport_recv_info memory"); ret = -ENOMEM; goto err_vport_recv_info; } adapter->vports = rte_zmalloc("vports", adapter->max_vport_nb * sizeof(*adapter->vports), 0); if (adapter->vports == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate vports memory"); ret = -ENOMEM; goto err_vports; } adapter->max_rxq_per_msg = (IDPF_DFLT_MBX_BUF_SIZE - sizeof(struct virtchnl2_config_rx_queues)) / sizeof(struct virtchnl2_rxq_info); adapter->max_txq_per_msg = (IDPF_DFLT_MBX_BUF_SIZE - sizeof(struct virtchnl2_config_tx_queues)) / sizeof(struct virtchnl2_txq_info); adapter->cur_vports = 0; adapter->cur_vport_nb = 0; adapter->used_vecs_num = 0; return ret; err_vports: rte_free(adapter->vport_recv_info); adapter->vport_recv_info = NULL; err_vport_recv_info: rte_free(adapter->vport_req_info); adapter->vport_req_info = NULL; err_caps: rte_free(adapter->caps); adapter->caps = NULL; err_api: rte_free(adapter->mbx_resp); adapter->mbx_resp = NULL; err_mbx: idpf_ctlq_deinit(hw); err: return ret; } static const struct eth_dev_ops idpf_eth_dev_ops = { .dev_configure = idpf_dev_configure, .dev_close = idpf_dev_close, .rx_queue_setup = idpf_rx_queue_setup, .tx_queue_setup = idpf_tx_queue_setup, .dev_infos_get = idpf_dev_info_get, .dev_start = idpf_dev_start, .dev_stop = idpf_dev_stop, .link_update = idpf_dev_link_update, .rx_queue_start = idpf_rx_queue_start, .tx_queue_start = idpf_tx_queue_start, .rx_queue_stop = idpf_rx_queue_stop, .tx_queue_stop = idpf_tx_queue_stop, .rx_queue_release = idpf_dev_rx_queue_release, .tx_queue_release = idpf_dev_tx_queue_release, .mtu_set = idpf_dev_mtu_set, .dev_supported_ptypes_get = idpf_dev_supported_ptypes_get, }; static uint16_t idpf_get_vport_idx(struct idpf_vport **vports, uint16_t max_vport_nb) { uint16_t vport_idx; uint16_t i; for (i = 0; i < max_vport_nb; i++) { if (vports[i] == NULL) break; } if (i == max_vport_nb) vport_idx = IDPF_INVALID_VPORT_IDX; else vport_idx = i; return vport_idx; } static int idpf_dev_init(struct rte_eth_dev *dev, void *init_params) { struct idpf_vport *vport = dev->data->dev_private; struct idpf_adapter *adapter = init_params; int ret = 0; dev->dev_ops = &idpf_eth_dev_ops; vport->adapter = adapter; ret = idpf_init_vport_req_info(dev); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to init vport req_info."); goto err; } ret = idpf_vc_create_vport(adapter); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to create vport."); goto err_create_vport; } ret = idpf_init_vport(dev); if (ret != 0) { PMD_INIT_LOG(ERR, "Failed to init vports."); goto err_init_vport; } adapter->cur_vport_idx = idpf_get_vport_idx(adapter->vports, adapter->max_vport_nb); dev->data->mac_addrs = rte_zmalloc(NULL, RTE_ETHER_ADDR_LEN, 0); if (dev->data->mac_addrs == NULL) { PMD_INIT_LOG(ERR, "Cannot allocate mac_addr memory."); ret = -ENOMEM; goto err_init_vport; } rte_ether_addr_copy((struct rte_ether_addr *)vport->default_mac_addr, &dev->data->mac_addrs[0]); return 0; err_init_vport: idpf_vc_destroy_vport(vport); err_create_vport: rte_free(vport->adapter->vport_req_info[vport->adapter->cur_vport_idx]); err: return ret; } static const struct rte_pci_id pci_id_idpf_map[] = { { RTE_PCI_DEVICE(IDPF_INTEL_VENDOR_ID, IDPF_DEV_ID_PF) }, { .vendor_id = 0, /* sentinel */ }, }; struct idpf_adapter * idpf_find_adapter(struct rte_pci_device *pci_dev) { struct idpf_adapter *adapter; int found = 0; if (pci_dev == NULL) return NULL; rte_spinlock_lock(&idpf_adapter_lock); TAILQ_FOREACH(adapter, &idpf_adapter_list, next) { if (strncmp(adapter->name, pci_dev->device.name, PCI_PRI_STR_SIZE) == 0) { found = 1; break; } } rte_spinlock_unlock(&idpf_adapter_lock); if (found == 0) return NULL; return adapter; } static void idpf_adapter_rel(struct idpf_adapter *adapter) { struct idpf_hw *hw = &adapter->hw; int i; idpf_ctlq_deinit(hw); rte_free(adapter->caps); adapter->caps = NULL; rte_free(adapter->mbx_resp); adapter->mbx_resp = NULL; if (adapter->vport_req_info != NULL) { for (i = 0; i < adapter->max_vport_nb; i++) { rte_free(adapter->vport_req_info[i]); adapter->vport_req_info[i] = NULL; } rte_free(adapter->vport_req_info); adapter->vport_req_info = NULL; } if (adapter->vport_recv_info != NULL) { for (i = 0; i < adapter->max_vport_nb; i++) { rte_free(adapter->vport_recv_info[i]); adapter->vport_recv_info[i] = NULL; } rte_free(adapter->vport_recv_info); adapter->vport_recv_info = NULL; } rte_free(adapter->vports); adapter->vports = NULL; } static int idpf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_device *pci_dev) { struct idpf_adapter *adapter; char name[RTE_ETH_NAME_MAX_LEN]; int i, retval; bool first_probe = false; if (!idpf_adapter_list_init) { rte_spinlock_init(&idpf_adapter_lock); TAILQ_INIT(&idpf_adapter_list); idpf_adapter_list_init = true; } adapter = idpf_find_adapter(pci_dev); if (adapter == NULL) { first_probe = true; adapter = rte_zmalloc("idpf_adapter", sizeof(struct idpf_adapter), 0); if (adapter == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate adapter."); return -ENOMEM; } retval = idpf_adapter_init(pci_dev, adapter); if (retval != 0) { PMD_INIT_LOG(ERR, "Failed to init adapter."); return retval; } rte_spinlock_lock(&idpf_adapter_lock); TAILQ_INSERT_TAIL(&idpf_adapter_list, adapter, next); rte_spinlock_unlock(&idpf_adapter_lock); } retval = idpf_parse_devargs(pci_dev, adapter); if (retval != 0) { PMD_INIT_LOG(ERR, "Failed to parse private devargs"); goto err; } if (adapter->req_vport_nb == 0) { /* If no vport devarg, create vport 0 by default. */ snprintf(name, sizeof(name), "idpf_%s_vport_0", pci_dev->device.name); retval = rte_eth_dev_create(&pci_dev->device, name, sizeof(struct idpf_vport), NULL, NULL, idpf_dev_init, adapter); if (retval != 0) PMD_DRV_LOG(ERR, "Failed to create default vport 0"); adapter->cur_vports |= RTE_BIT32(0); adapter->cur_vport_nb++; } else { for (i = 0; i < adapter->req_vport_nb; i++) { snprintf(name, sizeof(name), "idpf_%s_vport_%d", pci_dev->device.name, adapter->req_vports[i]); retval = rte_eth_dev_create(&pci_dev->device, name, sizeof(struct idpf_vport), NULL, NULL, idpf_dev_init, adapter); if (retval != 0) PMD_DRV_LOG(ERR, "Failed to create vport %d", adapter->req_vports[i]); } } return 0; err: if (first_probe) { rte_spinlock_lock(&idpf_adapter_lock); TAILQ_REMOVE(&idpf_adapter_list, adapter, next); rte_spinlock_unlock(&idpf_adapter_lock); idpf_adapter_rel(adapter); rte_free(adapter); } return retval; } static int idpf_pci_remove(struct rte_pci_device *pci_dev) { struct idpf_adapter *adapter = idpf_find_adapter(pci_dev); uint16_t port_id; /* Ethdev created can be found RTE_ETH_FOREACH_DEV_OF through rte_device */ RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device) { rte_eth_dev_close(port_id); } rte_spinlock_lock(&idpf_adapter_lock); TAILQ_REMOVE(&idpf_adapter_list, adapter, next); rte_spinlock_unlock(&idpf_adapter_lock); idpf_adapter_rel(adapter); rte_free(adapter); return 0; } static struct rte_pci_driver rte_idpf_pmd = { .id_table = pci_id_idpf_map, .drv_flags = RTE_PCI_DRV_NEED_MAPPING, .probe = idpf_pci_probe, .remove = idpf_pci_remove, }; /** * Driver initialization routine. * Invoked once at EAL init time. * Register itself as the [Poll Mode] Driver of PCI devices. */ RTE_PMD_REGISTER_PCI(net_idpf, rte_idpf_pmd); RTE_PMD_REGISTER_PCI_TABLE(net_idpf, pci_id_idpf_map); RTE_PMD_REGISTER_KMOD_DEP(net_idpf, "* igb_uio | vfio-pci"); RTE_PMD_REGISTER_PARAM_STRING(net_idpf, IDPF_TX_SINGLE_Q "=<0|1> " IDPF_RX_SINGLE_Q "=<0|1> " IDPF_VPORT "=[vport_set0,[vport_set1],...]"); RTE_LOG_REGISTER_SUFFIX(idpf_logtype_init, init, NOTICE); RTE_LOG_REGISTER_SUFFIX(idpf_logtype_driver, driver, NOTICE);