/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2020 Intel Corporation */ #include #include #include #include #include "iavf.h" #include "iavf_rxtx.h" #include "iavf_log.h" #include "iavf_generic_flow.h" #include "iavf_ipsec_crypto.h" #include "iavf_ipsec_crypto_capabilities.h" /** * iAVF IPsec Crypto Security Context */ struct iavf_security_ctx { struct iavf_adapter *adapter; int pkt_md_offset; struct rte_cryptodev_capabilities *crypto_capabilities; }; /** * iAVF IPsec Crypto Security Session Parameters */ struct iavf_security_session { struct iavf_adapter *adapter; enum rte_security_ipsec_sa_mode mode; enum rte_security_ipsec_tunnel_type type; enum rte_security_ipsec_sa_direction direction; struct { uint32_t spi; /* Security Parameter Index */ uint32_t hw_idx; /* SA Index in hardware table */ } sa; struct { uint8_t enabled :1; union { uint64_t value; struct { uint32_t hi; uint32_t low; }; }; } esn; struct { uint8_t enabled :1; } udp_encap; size_t iv_sz; size_t icv_sz; size_t block_sz; struct iavf_ipsec_crypto_pkt_metadata pkt_metadata_template; }; /** * IV Length field in IPsec Tx Desc uses the following encoding: * * 0B - 0 * 4B - 1 * 8B - 2 * 16B - 3 * * but we also need the IV Length for TSO to correctly calculate the total * header length so placing it in the upper 6-bits here for easier retrieval. */ static inline uint8_t calc_ipsec_desc_iv_len_field(uint16_t iv_sz) { uint8_t iv_length = IAVF_IPSEC_IV_LEN_NONE; switch (iv_sz) { case 4: iv_length = IAVF_IPSEC_IV_LEN_DW; break; case 8: iv_length = IAVF_IPSEC_IV_LEN_DDW; break; case 16: iv_length = IAVF_IPSEC_IV_LEN_QDW; break; } return (iv_sz << 2) | iv_length; } static unsigned int iavf_ipsec_crypto_session_size_get(void *device __rte_unused) { return sizeof(struct iavf_security_session); } static const struct rte_cryptodev_symmetric_capability * get_capability(struct iavf_security_ctx *iavf_sctx, uint32_t algo, uint32_t type) { const struct rte_cryptodev_capabilities *capability; int i = 0; capability = &iavf_sctx->crypto_capabilities[i]; while (capability->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { if (capability->op == RTE_CRYPTO_OP_TYPE_SYMMETRIC && (uint32_t)capability->sym.xform_type == type && (uint32_t)capability->sym.cipher.algo == algo) return &capability->sym; /** try next capability */ capability = &iavf_crypto_capabilities[i++]; } return NULL; } static const struct rte_cryptodev_symmetric_capability * get_auth_capability(struct iavf_security_ctx *iavf_sctx, enum rte_crypto_auth_algorithm algo) { return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_AUTH); } static const struct rte_cryptodev_symmetric_capability * get_cipher_capability(struct iavf_security_ctx *iavf_sctx, enum rte_crypto_cipher_algorithm algo) { return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_CIPHER); } static const struct rte_cryptodev_symmetric_capability * get_aead_capability(struct iavf_security_ctx *iavf_sctx, enum rte_crypto_aead_algorithm algo) { return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_AEAD); } static uint16_t get_cipher_blocksize(struct iavf_security_ctx *iavf_sctx, enum rte_crypto_cipher_algorithm algo) { const struct rte_cryptodev_symmetric_capability *capability; capability = get_cipher_capability(iavf_sctx, algo); if (capability == NULL) return 0; return capability->cipher.block_size; } static uint16_t get_aead_blocksize(struct iavf_security_ctx *iavf_sctx, enum rte_crypto_aead_algorithm algo) { const struct rte_cryptodev_symmetric_capability *capability; capability = get_aead_capability(iavf_sctx, algo); if (capability == NULL) return 0; return capability->cipher.block_size; } static uint16_t get_auth_blocksize(struct iavf_security_ctx *iavf_sctx, enum rte_crypto_auth_algorithm algo) { const struct rte_cryptodev_symmetric_capability *capability; capability = get_auth_capability(iavf_sctx, algo); if (capability == NULL) return 0; return capability->auth.block_size; } static uint8_t calc_context_desc_cipherblock_sz(size_t len) { switch (len) { case 8: return 0x2; case 16: return 0x3; default: return 0x0; } } static int valid_length(uint32_t len, uint32_t min, uint32_t max, uint32_t increment) { if (len < min || len > max) return false; if (increment == 0) return true; if ((len - min) % increment) return false; /* make sure it fits in the key array */ if (len > VIRTCHNL_IPSEC_MAX_KEY_LEN) return false; return true; } static int valid_auth_xform(struct iavf_security_ctx *iavf_sctx, struct rte_crypto_auth_xform *auth) { const struct rte_cryptodev_symmetric_capability *capability; capability = get_auth_capability(iavf_sctx, auth->algo); if (capability == NULL) return false; /* verify key size */ if (!valid_length(auth->key.length, capability->auth.key_size.min, capability->auth.key_size.max, capability->aead.key_size.increment)) return false; return true; } static int valid_cipher_xform(struct iavf_security_ctx *iavf_sctx, struct rte_crypto_cipher_xform *cipher) { const struct rte_cryptodev_symmetric_capability *capability; capability = get_cipher_capability(iavf_sctx, cipher->algo); if (capability == NULL) return false; /* verify key size */ if (!valid_length(cipher->key.length, capability->cipher.key_size.min, capability->cipher.key_size.max, capability->cipher.key_size.increment)) return false; return true; } static int valid_aead_xform(struct iavf_security_ctx *iavf_sctx, struct rte_crypto_aead_xform *aead) { const struct rte_cryptodev_symmetric_capability *capability; capability = get_aead_capability(iavf_sctx, aead->algo); if (capability == NULL) return false; /* verify key size */ if (!valid_length(aead->key.length, capability->aead.key_size.min, capability->aead.key_size.max, capability->aead.key_size.increment)) return false; return true; } static int iavf_ipsec_crypto_session_validate_conf(struct iavf_security_ctx *iavf_sctx, struct rte_security_session_conf *conf) { /** validate security action/protocol selection */ if (conf->action_type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO || conf->protocol != RTE_SECURITY_PROTOCOL_IPSEC) { PMD_DRV_LOG(ERR, "Invalid action / protocol specified"); return -EINVAL; } /** validate IPsec protocol selection */ if (conf->ipsec.proto != RTE_SECURITY_IPSEC_SA_PROTO_ESP) { PMD_DRV_LOG(ERR, "Invalid IPsec protocol specified"); return -EINVAL; } /** validate selected options */ if (conf->ipsec.options.copy_dscp || conf->ipsec.options.copy_flabel || conf->ipsec.options.copy_df || conf->ipsec.options.dec_ttl || conf->ipsec.options.ecn || conf->ipsec.options.stats) { PMD_DRV_LOG(ERR, "Invalid IPsec option specified"); return -EINVAL; } /** * Validate crypto xforms parameters. * * AEAD transforms can be used for either inbound/outbound IPsec SAs, * for non-AEAD crypto transforms we explicitly only support CIPHER/AUTH * for outbound and AUTH/CIPHER chained transforms for inbound IPsec. */ if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { if (!valid_aead_xform(iavf_sctx, &conf->crypto_xform->aead)) { PMD_DRV_LOG(ERR, "Invalid IPsec option specified"); return -EINVAL; } } else if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS && conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && conf->crypto_xform->next && conf->crypto_xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { if (!valid_cipher_xform(iavf_sctx, &conf->crypto_xform->cipher)) { PMD_DRV_LOG(ERR, "Invalid IPsec option specified"); return -EINVAL; } if (!valid_auth_xform(iavf_sctx, &conf->crypto_xform->next->auth)) { PMD_DRV_LOG(ERR, "Invalid IPsec option specified"); return -EINVAL; } } else if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS && conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && conf->crypto_xform->next && conf->crypto_xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { if (!valid_auth_xform(iavf_sctx, &conf->crypto_xform->auth)) { PMD_DRV_LOG(ERR, "Invalid IPsec option specified"); return -EINVAL; } if (!valid_cipher_xform(iavf_sctx, &conf->crypto_xform->next->cipher)) { PMD_DRV_LOG(ERR, "Invalid IPsec option specified"); return -EINVAL; } } return 0; } static void sa_add_set_aead_params(struct virtchnl_ipsec_crypto_cfg_item *cfg, struct rte_crypto_aead_xform *aead, uint32_t salt) { cfg->crypto_type = VIRTCHNL_AEAD; switch (aead->algo) { case RTE_CRYPTO_AEAD_AES_CCM: cfg->algo_type = VIRTCHNL_AES_CCM; break; case RTE_CRYPTO_AEAD_AES_GCM: cfg->algo_type = VIRTCHNL_AES_GCM; break; case RTE_CRYPTO_AEAD_CHACHA20_POLY1305: cfg->algo_type = VIRTCHNL_CHACHA20_POLY1305; break; default: PMD_DRV_LOG(ERR, "Invalid AEAD parameters"); break; } cfg->key_len = aead->key.length; cfg->iv_len = sizeof(uint64_t); /* iv.length includes salt len */ cfg->digest_len = aead->digest_length; cfg->salt = salt; memcpy(cfg->key_data, aead->key.data, cfg->key_len); } static void sa_add_set_cipher_params(struct virtchnl_ipsec_crypto_cfg_item *cfg, struct rte_crypto_cipher_xform *cipher, uint32_t salt) { cfg->crypto_type = VIRTCHNL_CIPHER; switch (cipher->algo) { case RTE_CRYPTO_CIPHER_AES_CBC: cfg->algo_type = VIRTCHNL_AES_CBC; break; case RTE_CRYPTO_CIPHER_3DES_CBC: cfg->algo_type = VIRTCHNL_3DES_CBC; break; case RTE_CRYPTO_CIPHER_NULL: cfg->algo_type = VIRTCHNL_CIPHER_NO_ALG; break; case RTE_CRYPTO_CIPHER_AES_CTR: cfg->algo_type = VIRTCHNL_AES_CTR; cfg->salt = salt; break; default: PMD_DRV_LOG(ERR, "Invalid cipher parameters"); break; } cfg->key_len = cipher->key.length; cfg->iv_len = cipher->iv.length; cfg->salt = salt; memcpy(cfg->key_data, cipher->key.data, cfg->key_len); } static void sa_add_set_auth_params(struct virtchnl_ipsec_crypto_cfg_item *cfg, struct rte_crypto_auth_xform *auth, uint32_t salt) { cfg->crypto_type = VIRTCHNL_AUTH; switch (auth->algo) { case RTE_CRYPTO_AUTH_NULL: cfg->algo_type = VIRTCHNL_HASH_NO_ALG; break; case RTE_CRYPTO_AUTH_AES_CBC_MAC: cfg->algo_type = VIRTCHNL_AES_CBC_MAC; break; case RTE_CRYPTO_AUTH_AES_CMAC: cfg->algo_type = VIRTCHNL_AES_CMAC; break; case RTE_CRYPTO_AUTH_AES_XCBC_MAC: cfg->algo_type = VIRTCHNL_AES_XCBC_MAC; break; case RTE_CRYPTO_AUTH_MD5_HMAC: cfg->algo_type = VIRTCHNL_MD5_HMAC; break; case RTE_CRYPTO_AUTH_SHA1_HMAC: cfg->algo_type = VIRTCHNL_SHA1_HMAC; break; case RTE_CRYPTO_AUTH_SHA224_HMAC: cfg->algo_type = VIRTCHNL_SHA224_HMAC; break; case RTE_CRYPTO_AUTH_SHA256_HMAC: cfg->algo_type = VIRTCHNL_SHA256_HMAC; break; case RTE_CRYPTO_AUTH_SHA384_HMAC: cfg->algo_type = VIRTCHNL_SHA384_HMAC; break; case RTE_CRYPTO_AUTH_SHA512_HMAC: cfg->algo_type = VIRTCHNL_SHA512_HMAC; break; case RTE_CRYPTO_AUTH_AES_GMAC: cfg->algo_type = VIRTCHNL_AES_GMAC; cfg->salt = salt; break; default: PMD_DRV_LOG(ERR, "Invalid auth parameters"); break; } cfg->key_len = auth->key.length; /* special case for RTE_CRYPTO_AUTH_AES_GMAC */ if (auth->algo == RTE_CRYPTO_AUTH_AES_GMAC) cfg->iv_len = sizeof(uint64_t); /* iv.length includes salt */ else cfg->iv_len = auth->iv.length; cfg->digest_len = auth->digest_length; memcpy(cfg->key_data, auth->key.data, cfg->key_len); } /** * Send SA add virtual channel request to Inline IPsec driver. * * Inline IPsec driver expects SPI and destination IP address to be in host * order, but DPDK APIs are network order, therefore we need to do a htonl * conversion of these parameters. */ static uint32_t iavf_ipsec_crypto_security_association_add(struct iavf_adapter *adapter, struct rte_security_session_conf *conf) { struct inline_ipsec_msg *request = NULL, *response = NULL; struct virtchnl_ipsec_sa_cfg *sa_cfg; size_t request_len, response_len; int rc; request_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_sa_cfg); request = rte_malloc("iavf-sad-add-request", request_len, 0); if (request == NULL) { rc = -ENOMEM; goto update_cleanup; } response_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_sa_cfg_resp); response = rte_malloc("iavf-sad-add-response", response_len, 0); if (response == NULL) { rc = -ENOMEM; goto update_cleanup; } /* set msg header params */ request->ipsec_opcode = INLINE_IPSEC_OP_SA_CREATE; request->req_id = (uint16_t)0xDEADBEEF; /* set SA configuration params */ sa_cfg = (struct virtchnl_ipsec_sa_cfg *)(request + 1); sa_cfg->spi = conf->ipsec.spi; sa_cfg->virtchnl_protocol_type = VIRTCHNL_PROTO_ESP; sa_cfg->virtchnl_direction = conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS ? VIRTCHNL_DIR_INGRESS : VIRTCHNL_DIR_EGRESS; if (conf->ipsec.options.esn) { sa_cfg->esn_enabled = 1; sa_cfg->esn_hi = conf->ipsec.esn.hi; sa_cfg->esn_low = conf->ipsec.esn.low; } if (conf->ipsec.options.udp_encap) sa_cfg->udp_encap_enabled = 1; /* Set outer IP params */ if (conf->ipsec.tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) { sa_cfg->virtchnl_ip_type = VIRTCHNL_IPV4; *((uint32_t *)sa_cfg->dst_addr) = htonl(conf->ipsec.tunnel.ipv4.dst_ip.s_addr); } else { uint32_t *v6_dst_addr = (uint32_t *)conf->ipsec.tunnel.ipv6.dst_addr.s6_addr; sa_cfg->virtchnl_ip_type = VIRTCHNL_IPV6; ((uint32_t *)sa_cfg->dst_addr)[0] = htonl(v6_dst_addr[0]); ((uint32_t *)sa_cfg->dst_addr)[1] = htonl(v6_dst_addr[1]); ((uint32_t *)sa_cfg->dst_addr)[2] = htonl(v6_dst_addr[2]); ((uint32_t *)sa_cfg->dst_addr)[3] = htonl(v6_dst_addr[3]); } /* set crypto params */ if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { sa_add_set_aead_params(&sa_cfg->crypto_cfg.items[0], &conf->crypto_xform->aead, conf->ipsec.salt); } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { sa_add_set_cipher_params(&sa_cfg->crypto_cfg.items[0], &conf->crypto_xform->cipher, conf->ipsec.salt); sa_add_set_auth_params(&sa_cfg->crypto_cfg.items[1], &conf->crypto_xform->next->auth, conf->ipsec.salt); } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { sa_add_set_auth_params(&sa_cfg->crypto_cfg.items[0], &conf->crypto_xform->auth, conf->ipsec.salt); if (conf->crypto_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC) sa_add_set_cipher_params(&sa_cfg->crypto_cfg.items[1], &conf->crypto_xform->next->cipher, conf->ipsec.salt); } /* send virtual channel request to add SA to hardware database */ rc = iavf_ipsec_crypto_request(adapter, (uint8_t *)request, request_len, (uint8_t *)response, response_len); if (rc) goto update_cleanup; /* verify response id */ if (response->ipsec_opcode != request->ipsec_opcode || response->req_id != request->req_id) rc = -EFAULT; else rc = response->ipsec_data.sa_cfg_resp->sa_handle; update_cleanup: rte_free(response); rte_free(request); return rc; } static void set_pkt_metadata_template(struct iavf_ipsec_crypto_pkt_metadata *template, struct iavf_security_session *sess) { template->sa_idx = sess->sa.hw_idx; if (sess->udp_encap.enabled) template->ol_flags = IAVF_IPSEC_CRYPTO_OL_FLAGS_NATT; if (sess->esn.enabled) template->ol_flags = IAVF_IPSEC_CRYPTO_OL_FLAGS_ESN; template->len_iv = calc_ipsec_desc_iv_len_field(sess->iv_sz); template->ctx_desc_ipsec_params = calc_context_desc_cipherblock_sz(sess->block_sz) | ((uint8_t)(sess->icv_sz >> 2) << 3); } static void set_session_parameter(struct iavf_security_ctx *iavf_sctx, struct iavf_security_session *sess, struct rte_security_session_conf *conf, uint32_t sa_idx) { sess->adapter = iavf_sctx->adapter; sess->mode = conf->ipsec.mode; sess->direction = conf->ipsec.direction; if (sess->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL) sess->type = conf->ipsec.tunnel.type; sess->sa.spi = conf->ipsec.spi; sess->sa.hw_idx = sa_idx; if (conf->ipsec.options.esn) { sess->esn.enabled = 1; sess->esn.value = conf->ipsec.esn.value; } if (conf->ipsec.options.udp_encap) sess->udp_encap.enabled = 1; if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { sess->block_sz = get_aead_blocksize(iavf_sctx, conf->crypto_xform->aead.algo); sess->iv_sz = sizeof(uint64_t); /* iv.length includes salt */ sess->icv_sz = conf->crypto_xform->aead.digest_length; } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { sess->block_sz = get_cipher_blocksize(iavf_sctx, conf->crypto_xform->cipher.algo); sess->iv_sz = conf->crypto_xform->cipher.iv.length; sess->icv_sz = conf->crypto_xform->next->auth.digest_length; } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { if (conf->crypto_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { sess->block_sz = get_auth_blocksize(iavf_sctx, conf->crypto_xform->auth.algo); sess->iv_sz = sizeof(uint64_t); /* iv len inc. salt */ sess->icv_sz = conf->crypto_xform->auth.digest_length; } else { sess->block_sz = get_cipher_blocksize(iavf_sctx, conf->crypto_xform->next->cipher.algo); sess->iv_sz = conf->crypto_xform->next->cipher.iv.length; sess->icv_sz = conf->crypto_xform->auth.digest_length; } } set_pkt_metadata_template(&sess->pkt_metadata_template, sess); } /** * Create IPsec Security Association for inline IPsec Crypto offload. * * 1. validate session configuration parameters * 2. allocate session memory from mempool * 3. add SA to hardware database * 4. set session parameters * 5. create packet metadata template for datapath */ static int iavf_ipsec_crypto_session_create(void *device, struct rte_security_session_conf *conf, struct rte_security_session *session, struct rte_mempool *mempool) { struct rte_eth_dev *ethdev = device; struct iavf_adapter *adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private); struct iavf_security_ctx *iavf_sctx = adapter->security_ctx; struct iavf_security_session *iavf_session = NULL; int sa_idx; int ret = 0; /* validate that all SA parameters are valid for device */ ret = iavf_ipsec_crypto_session_validate_conf(iavf_sctx, conf); if (ret) return ret; /* allocate session context */ if (rte_mempool_get(mempool, (void **)&iavf_session)) { PMD_DRV_LOG(ERR, "Cannot get object from sess mempool"); return -ENOMEM; } /* add SA to hardware database */ sa_idx = iavf_ipsec_crypto_security_association_add(adapter, conf); if (sa_idx < 0) { PMD_DRV_LOG(ERR, "Failed to add SA (spi: %d, mode: %s, direction: %s)", conf->ipsec.spi, conf->ipsec.mode == RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT ? "transport" : "tunnel", conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS ? "inbound" : "outbound"); rte_mempool_put(mempool, iavf_session); return -EFAULT; } /* save data plane required session parameters */ set_session_parameter(iavf_sctx, iavf_session, conf, sa_idx); /* save to security session private data */ set_sec_session_private_data(session, iavf_session); return 0; } /** * Check if valid ipsec crypto action. * SPI must be non-zero and SPI in session must match SPI value * passed into function. * * returns: 0 if invalid session or SPI value equal zero * returns: 1 if valid */ uint32_t iavf_ipsec_crypto_action_valid(struct rte_eth_dev *ethdev, const struct rte_security_session *session, uint32_t spi) { struct iavf_adapter *adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private); struct iavf_security_session *sess = session->sess_private_data; /* verify we have a valid session and that it belong to this adapter */ if (unlikely(sess == NULL || sess->adapter != adapter)) return false; /* SPI value must be non-zero and must match flow SPI*/ if (spi == 0 || (htonl(sess->sa.spi) != spi)) return false; return true; } /** * Send virtual channel security policy add request to IES driver. * * IES driver expects SPI and destination IP address to be in host * order, but DPDK APIs are network order, therefore we need to do a htonl * conversion of these parameters. */ int iavf_ipsec_crypto_inbound_security_policy_add(struct iavf_adapter *adapter, uint32_t esp_spi, uint8_t is_v4, rte_be32_t v4_dst_addr, uint8_t *v6_dst_addr, uint8_t drop, bool is_udp, uint16_t udp_port) { struct inline_ipsec_msg *request = NULL, *response = NULL; size_t request_len, response_len; int rc = 0; request_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_sp_cfg); request = rte_malloc("iavf-inbound-security-policy-add-request", request_len, 0); if (request == NULL) { rc = -ENOMEM; goto update_cleanup; } /* set msg header params */ request->ipsec_opcode = INLINE_IPSEC_OP_SP_CREATE; request->req_id = (uint16_t)0xDEADBEEF; /* ESP SPI */ request->ipsec_data.sp_cfg->spi = htonl(esp_spi); /* Destination IP */ if (is_v4) { request->ipsec_data.sp_cfg->table_id = VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV4; request->ipsec_data.sp_cfg->dip[0] = htonl(v4_dst_addr); } else { request->ipsec_data.sp_cfg->table_id = VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV6; request->ipsec_data.sp_cfg->dip[0] = htonl(((uint32_t *)v6_dst_addr)[0]); request->ipsec_data.sp_cfg->dip[1] = htonl(((uint32_t *)v6_dst_addr)[1]); request->ipsec_data.sp_cfg->dip[2] = htonl(((uint32_t *)v6_dst_addr)[2]); request->ipsec_data.sp_cfg->dip[3] = htonl(((uint32_t *)v6_dst_addr)[3]); } request->ipsec_data.sp_cfg->drop = drop; /** Traffic Class/Congestion Domain currently not support */ request->ipsec_data.sp_cfg->set_tc = 0; request->ipsec_data.sp_cfg->cgd = 0; request->ipsec_data.sp_cfg->is_udp = is_udp; request->ipsec_data.sp_cfg->udp_port = htons(udp_port); response_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_sp_cfg_resp); response = rte_malloc("iavf-inbound-security-policy-add-response", response_len, 0); if (response == NULL) { rc = -ENOMEM; goto update_cleanup; } /* send virtual channel request to add SA to hardware database */ rc = iavf_ipsec_crypto_request(adapter, (uint8_t *)request, request_len, (uint8_t *)response, response_len); if (rc) goto update_cleanup; /* verify response */ if (response->ipsec_opcode != request->ipsec_opcode || response->req_id != request->req_id) rc = -EFAULT; else rc = response->ipsec_data.sp_cfg_resp->rule_id; update_cleanup: rte_free(request); rte_free(response); return rc; } static uint32_t iavf_ipsec_crypto_sa_update_esn(struct iavf_adapter *adapter, struct iavf_security_session *sess) { struct inline_ipsec_msg *request = NULL, *response = NULL; size_t request_len, response_len; int rc = 0; request_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_sa_update); request = rte_malloc("iavf-sa-update-request", request_len, 0); if (request == NULL) { rc = -ENOMEM; goto update_cleanup; } response_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_resp); response = rte_malloc("iavf-sa-update-response", response_len, 0); if (response == NULL) { rc = -ENOMEM; goto update_cleanup; } /* set msg header params */ request->ipsec_opcode = INLINE_IPSEC_OP_SA_UPDATE; request->req_id = (uint16_t)0xDEADBEEF; /* set request params */ request->ipsec_data.sa_update->sa_index = sess->sa.hw_idx; request->ipsec_data.sa_update->esn_hi = sess->esn.hi; /* send virtual channel request to add SA to hardware database */ rc = iavf_ipsec_crypto_request(adapter, (uint8_t *)request, request_len, (uint8_t *)response, response_len); if (rc) goto update_cleanup; /* verify response */ if (response->ipsec_opcode != request->ipsec_opcode || response->req_id != request->req_id) rc = -EFAULT; else rc = response->ipsec_data.ipsec_resp->resp; update_cleanup: rte_free(request); rte_free(response); return rc; } static int iavf_ipsec_crypto_session_update(void *device, struct rte_security_session *session, struct rte_security_session_conf *conf) { struct iavf_adapter *adapter = NULL; struct iavf_security_session *iavf_sess = NULL; struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device; int rc = 0; adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); iavf_sess = (struct iavf_security_session *)session->sess_private_data; /* verify we have a valid session and that it belong to this adapter */ if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter)) return -EINVAL; /* update esn hi 32-bits */ if (iavf_sess->esn.enabled && conf->ipsec.options.esn) { /** * Update ESN in hardware for inbound SA. Store in * iavf_security_session for outbound SA for use * in *iavf_ipsec_crypto_pkt_metadata_set* function. */ iavf_sess->esn.hi = conf->ipsec.esn.hi; iavf_sess->esn.low = conf->ipsec.esn.low; if (iavf_sess->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) rc = iavf_ipsec_crypto_sa_update_esn(adapter, iavf_sess); } return rc; } static int iavf_ipsec_crypto_session_stats_get(void *device __rte_unused, struct rte_security_session *session __rte_unused, struct rte_security_stats *stats __rte_unused) { return -EOPNOTSUPP; } int iavf_ipsec_crypto_security_policy_delete(struct iavf_adapter *adapter, uint8_t is_v4, uint32_t flow_id) { struct inline_ipsec_msg *request = NULL, *response = NULL; size_t request_len, response_len; int rc = 0; request_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_sp_destroy); request = rte_malloc("iavf-sp-del-request", request_len, 0); if (request == NULL) { rc = -ENOMEM; goto update_cleanup; } response_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_resp); response = rte_malloc("iavf-sp-del-response", response_len, 0); if (response == NULL) { rc = -ENOMEM; goto update_cleanup; } /* set msg header params */ request->ipsec_opcode = INLINE_IPSEC_OP_SP_DESTROY; request->req_id = (uint16_t)0xDEADBEEF; /* set security policy params */ request->ipsec_data.sp_destroy->table_id = is_v4 ? VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV4 : VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV6; request->ipsec_data.sp_destroy->rule_id = flow_id; /* send virtual channel request to add SA to hardware database */ rc = iavf_ipsec_crypto_request(adapter, (uint8_t *)request, request_len, (uint8_t *)response, response_len); if (rc) goto update_cleanup; /* verify response */ if (response->ipsec_opcode != request->ipsec_opcode || response->req_id != request->req_id) rc = -EFAULT; else return response->ipsec_data.ipsec_status->status; update_cleanup: rte_free(request); rte_free(response); return rc; } static uint32_t iavf_ipsec_crypto_sa_del(struct iavf_adapter *adapter, struct iavf_security_session *sess) { struct inline_ipsec_msg *request = NULL, *response = NULL; size_t request_len, response_len; int rc = 0; request_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_sa_destroy); request = rte_malloc("iavf-sa-del-request", request_len, 0); if (request == NULL) { rc = -ENOMEM; goto update_cleanup; } response_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_resp); response = rte_malloc("iavf-sa-del-response", response_len, 0); if (response == NULL) { rc = -ENOMEM; goto update_cleanup; } /* set msg header params */ request->ipsec_opcode = INLINE_IPSEC_OP_SA_DESTROY; request->req_id = (uint16_t)0xDEADBEEF; /** * SA delete supports deletion of 1-8 specified SA's or if the flag * field is zero, all SA's associated with VF will be deleted. */ if (sess) { request->ipsec_data.sa_destroy->flag = 0x1; request->ipsec_data.sa_destroy->sa_index[0] = sess->sa.hw_idx; } else { request->ipsec_data.sa_destroy->flag = 0x0; } /* send virtual channel request to add SA to hardware database */ rc = iavf_ipsec_crypto_request(adapter, (uint8_t *)request, request_len, (uint8_t *)response, response_len); if (rc) goto update_cleanup; /* verify response */ if (response->ipsec_opcode != request->ipsec_opcode || response->req_id != request->req_id) rc = -EFAULT; /** * Delete status will be the same bitmask as sa_destroy request flag if * deletes successful */ if (request->ipsec_data.sa_destroy->flag != response->ipsec_data.ipsec_status->status) rc = -EFAULT; update_cleanup: rte_free(response); rte_free(request); return rc; } static int iavf_ipsec_crypto_session_destroy(void *device, struct rte_security_session *session) { struct iavf_adapter *adapter = NULL; struct iavf_security_session *iavf_sess = NULL; struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device; int ret; adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); iavf_sess = (struct iavf_security_session *)session->sess_private_data; /* verify we have a valid session and that it belong to this adapter */ if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter)) return -EINVAL; ret = iavf_ipsec_crypto_sa_del(adapter, iavf_sess); rte_mempool_put(rte_mempool_from_obj(iavf_sess), (void *)iavf_sess); return ret; } /** * Get ESP trailer from packet as well as calculate the total ESP trailer * length, which include padding, ESP trailer footer and the ICV */ static inline struct rte_esp_tail * iavf_ipsec_crypto_get_esp_trailer(struct rte_mbuf *m, struct iavf_security_session *s, uint16_t *esp_trailer_length) { struct rte_esp_tail *esp_trailer; uint16_t length = sizeof(struct rte_esp_tail) + s->icv_sz; uint16_t offset = 0; /** * The ICV will not be present in TSO packets as this is appended by * hardware during segment generation */ if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) length -= s->icv_sz; *esp_trailer_length = length; /** * Calculate offset in packet to ESP trailer header, this should be * total packet length less the size of the ESP trailer plus the ICV * length if it is present */ offset = rte_pktmbuf_pkt_len(m) - length; if (m->nb_segs > 1) { /* find segment which esp trailer is located */ while (m->data_len < offset) { offset -= m->data_len; m = m->next; } } esp_trailer = rte_pktmbuf_mtod_offset(m, struct rte_esp_tail *, offset); *esp_trailer_length += esp_trailer->pad_len; return esp_trailer; } static inline uint16_t iavf_ipsec_crypto_compute_l4_payload_length(struct rte_mbuf *m, struct iavf_security_session *s, uint16_t esp_tlen) { uint16_t ol2_len = m->l2_len; /* MAC + VLAN */ uint16_t ol3_len = 0; /* ipv4/6 + ext hdrs */ uint16_t ol4_len = 0; /* UDP NATT */ uint16_t l3_len = 0; /* IPv4/6 + ext hdrs */ uint16_t l4_len = 0; /* TCP/UDP/STCP hdrs */ uint16_t esp_hlen = sizeof(struct rte_esp_hdr) + s->iv_sz; if (s->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL) ol3_len = m->outer_l3_len; /**< * application provided l3len assumed to include length of * ipv4/6 hdr + ext hdrs */ if (s->udp_encap.enabled) { ol4_len = sizeof(struct rte_udp_hdr); l3_len = m->l3_len - ol4_len; l4_len = l3_len; } else { l3_len = m->l3_len; l4_len = m->l4_len; } return rte_pktmbuf_pkt_len(m) - (ol2_len + ol3_len + ol4_len + esp_hlen + l3_len + l4_len + esp_tlen); } static int iavf_ipsec_crypto_pkt_metadata_set(void *device, struct rte_security_session *session, struct rte_mbuf *m, void *params) { struct rte_eth_dev *ethdev = device; struct iavf_adapter *adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private); struct iavf_security_ctx *iavf_sctx = adapter->security_ctx; struct iavf_security_session *iavf_sess = session->sess_private_data; struct iavf_ipsec_crypto_pkt_metadata *md; struct rte_esp_tail *esp_tail; uint64_t *sqn = params; uint16_t esp_trailer_length; /* Check we have valid session and is associated with this device */ if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter)) return -EINVAL; /* Get dynamic metadata location from mbuf */ md = RTE_MBUF_DYNFIELD(m, iavf_sctx->pkt_md_offset, struct iavf_ipsec_crypto_pkt_metadata *); /* Set immutable metadata values from session template */ memcpy(md, &iavf_sess->pkt_metadata_template, sizeof(struct iavf_ipsec_crypto_pkt_metadata)); esp_tail = iavf_ipsec_crypto_get_esp_trailer(m, iavf_sess, &esp_trailer_length); /* Set per packet mutable metadata values */ md->esp_trailer_len = esp_trailer_length; md->l4_payload_len = iavf_ipsec_crypto_compute_l4_payload_length(m, iavf_sess, esp_trailer_length); md->next_proto = esp_tail->next_proto; /* If Extended SN in use set the upper 32-bits in metadata */ if (iavf_sess->esn.enabled && sqn != NULL) md->esn = (uint32_t)(*sqn >> 32); return 0; } static int iavf_ipsec_crypto_device_capabilities_get(struct iavf_adapter *adapter, struct virtchnl_ipsec_cap *capability) { /* Perform pf-vf comms */ struct inline_ipsec_msg *request = NULL, *response = NULL; size_t request_len, response_len; int rc; request_len = sizeof(struct inline_ipsec_msg); request = rte_malloc("iavf-device-capability-request", request_len, 0); if (request == NULL) { rc = -ENOMEM; goto update_cleanup; } response_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_cap); response = rte_malloc("iavf-device-capability-response", response_len, 0); if (response == NULL) { rc = -ENOMEM; goto update_cleanup; } /* set msg header params */ request->ipsec_opcode = INLINE_IPSEC_OP_GET_CAP; request->req_id = (uint16_t)0xDEADBEEF; /* send virtual channel request to add SA to hardware database */ rc = iavf_ipsec_crypto_request(adapter, (uint8_t *)request, request_len, (uint8_t *)response, response_len); if (rc) goto update_cleanup; /* verify response id */ if (response->ipsec_opcode != request->ipsec_opcode || response->req_id != request->req_id){ rc = -EFAULT; goto update_cleanup; } memcpy(capability, response->ipsec_data.ipsec_cap, sizeof(*capability)); update_cleanup: rte_free(response); rte_free(request); return rc; } enum rte_crypto_auth_algorithm auth_maptbl[] = { /* Hash Algorithm */ [VIRTCHNL_HASH_NO_ALG] = RTE_CRYPTO_AUTH_NULL, [VIRTCHNL_AES_CBC_MAC] = RTE_CRYPTO_AUTH_AES_CBC_MAC, [VIRTCHNL_AES_CMAC] = RTE_CRYPTO_AUTH_AES_CMAC, [VIRTCHNL_AES_GMAC] = RTE_CRYPTO_AUTH_AES_GMAC, [VIRTCHNL_AES_XCBC_MAC] = RTE_CRYPTO_AUTH_AES_XCBC_MAC, [VIRTCHNL_MD5_HMAC] = RTE_CRYPTO_AUTH_MD5_HMAC, [VIRTCHNL_SHA1_HMAC] = RTE_CRYPTO_AUTH_SHA1_HMAC, [VIRTCHNL_SHA224_HMAC] = RTE_CRYPTO_AUTH_SHA224_HMAC, [VIRTCHNL_SHA256_HMAC] = RTE_CRYPTO_AUTH_SHA256_HMAC, [VIRTCHNL_SHA384_HMAC] = RTE_CRYPTO_AUTH_SHA384_HMAC, [VIRTCHNL_SHA512_HMAC] = RTE_CRYPTO_AUTH_SHA512_HMAC, [VIRTCHNL_SHA3_224_HMAC] = RTE_CRYPTO_AUTH_SHA3_224_HMAC, [VIRTCHNL_SHA3_256_HMAC] = RTE_CRYPTO_AUTH_SHA3_256_HMAC, [VIRTCHNL_SHA3_384_HMAC] = RTE_CRYPTO_AUTH_SHA3_384_HMAC, [VIRTCHNL_SHA3_512_HMAC] = RTE_CRYPTO_AUTH_SHA3_512_HMAC, }; static void update_auth_capabilities(struct rte_cryptodev_capabilities *scap, struct virtchnl_algo_cap *acap) { struct rte_cryptodev_symmetric_capability *capability = &scap->sym; scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC; capability->xform_type = RTE_CRYPTO_SYM_XFORM_AUTH; capability->auth.algo = auth_maptbl[acap->algo_type]; capability->auth.block_size = acap->block_size; capability->auth.key_size.min = acap->min_key_size; capability->auth.key_size.max = acap->max_key_size; capability->auth.key_size.increment = acap->inc_key_size; capability->auth.digest_size.min = acap->min_digest_size; capability->auth.digest_size.max = acap->max_digest_size; capability->auth.digest_size.increment = acap->inc_digest_size; } enum rte_crypto_cipher_algorithm cipher_maptbl[] = { /* Cipher Algorithm */ [VIRTCHNL_CIPHER_NO_ALG] = RTE_CRYPTO_CIPHER_NULL, [VIRTCHNL_3DES_CBC] = RTE_CRYPTO_CIPHER_3DES_CBC, [VIRTCHNL_AES_CBC] = RTE_CRYPTO_CIPHER_AES_CBC, [VIRTCHNL_AES_CTR] = RTE_CRYPTO_CIPHER_AES_CTR, }; static void update_cipher_capabilities(struct rte_cryptodev_capabilities *scap, struct virtchnl_algo_cap *acap) { struct rte_cryptodev_symmetric_capability *capability = &scap->sym; scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC; capability->xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER; capability->cipher.algo = cipher_maptbl[acap->algo_type]; capability->cipher.block_size = acap->block_size; capability->cipher.key_size.min = acap->min_key_size; capability->cipher.key_size.max = acap->max_key_size; capability->cipher.key_size.increment = acap->inc_key_size; capability->cipher.iv_size.min = acap->min_iv_size; capability->cipher.iv_size.max = acap->max_iv_size; capability->cipher.iv_size.increment = acap->inc_iv_size; } enum rte_crypto_aead_algorithm aead_maptbl[] = { /* AEAD Algorithm */ [VIRTCHNL_AES_CCM] = RTE_CRYPTO_AEAD_AES_CCM, [VIRTCHNL_AES_GCM] = RTE_CRYPTO_AEAD_AES_GCM, [VIRTCHNL_CHACHA20_POLY1305] = RTE_CRYPTO_AEAD_CHACHA20_POLY1305, }; static void update_aead_capabilities(struct rte_cryptodev_capabilities *scap, struct virtchnl_algo_cap *acap) { struct rte_cryptodev_symmetric_capability *capability = &scap->sym; scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC; capability->xform_type = RTE_CRYPTO_SYM_XFORM_AEAD; capability->aead.algo = aead_maptbl[acap->algo_type]; capability->aead.block_size = acap->block_size; capability->aead.key_size.min = acap->min_key_size; capability->aead.key_size.max = acap->max_key_size; capability->aead.key_size.increment = acap->inc_key_size; capability->aead.aad_size.min = acap->min_aad_size; capability->aead.aad_size.max = acap->max_aad_size; capability->aead.aad_size.increment = acap->inc_aad_size; capability->aead.iv_size.min = acap->min_iv_size; capability->aead.iv_size.max = acap->max_iv_size; capability->aead.iv_size.increment = acap->inc_iv_size; capability->aead.digest_size.min = acap->min_digest_size; capability->aead.digest_size.max = acap->max_digest_size; capability->aead.digest_size.increment = acap->inc_digest_size; } /** * Dynamically set crypto capabilities based on virtchannel IPsec * capabilities structure. */ int iavf_ipsec_crypto_set_security_capabililites(struct iavf_security_ctx *iavf_sctx, struct virtchnl_ipsec_cap *vch_cap) { struct rte_cryptodev_capabilities *capabilities; int i, j, number_of_capabilities = 0, ci = 0; /* Count the total number of crypto algorithms supported */ for (i = 0; i < VIRTCHNL_IPSEC_MAX_CRYPTO_CAP_NUM; i++) number_of_capabilities += vch_cap->cap[i].algo_cap_num; /** * Allocate cryptodev capabilities structure for * *number_of_capabilities* items plus one item to null terminate the * array */ capabilities = rte_zmalloc("crypto_cap", sizeof(struct rte_cryptodev_capabilities) * (number_of_capabilities + 1), 0); if (!capabilities) return -ENOMEM; capabilities[number_of_capabilities].op = RTE_CRYPTO_OP_TYPE_UNDEFINED; /** * Iterate over each virtchnl crypto capability by crypto type and * algorithm. */ for (i = 0; i < VIRTCHNL_IPSEC_MAX_CRYPTO_CAP_NUM; i++) { for (j = 0; j < vch_cap->cap[i].algo_cap_num; j++, ci++) { switch (vch_cap->cap[i].crypto_type) { case VIRTCHNL_AUTH: update_auth_capabilities(&capabilities[ci], &vch_cap->cap[i].algo_cap_list[j]); break; case VIRTCHNL_CIPHER: update_cipher_capabilities(&capabilities[ci], &vch_cap->cap[i].algo_cap_list[j]); break; case VIRTCHNL_AEAD: update_aead_capabilities(&capabilities[ci], &vch_cap->cap[i].algo_cap_list[j]); break; default: capabilities[ci].op = RTE_CRYPTO_OP_TYPE_UNDEFINED; break; } } } iavf_sctx->crypto_capabilities = capabilities; return 0; } /** * Get security capabilities for device */ static const struct rte_security_capability * iavf_ipsec_crypto_capabilities_get(void *device) { struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device; struct iavf_adapter *adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); struct iavf_security_ctx *iavf_sctx = adapter->security_ctx; unsigned int i; static struct rte_security_capability iavf_security_capabilities[] = { { /* IPsec Inline Crypto ESP Tunnel Egress */ .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, .protocol = RTE_SECURITY_PROTOCOL_IPSEC, .ipsec = { .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS, .options = { .udp_encap = 1, .stats = 1, .esn = 1 }, }, .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA }, { /* IPsec Inline Crypto ESP Tunnel Ingress */ .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, .protocol = RTE_SECURITY_PROTOCOL_IPSEC, .ipsec = { .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS, .options = { .udp_encap = 1, .stats = 1, .esn = 1 }, }, .ol_flags = 0 }, { /* IPsec Inline Crypto ESP Transport Egress */ .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, .protocol = RTE_SECURITY_PROTOCOL_IPSEC, .ipsec = { .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT, .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS, .options = { .udp_encap = 1, .stats = 1, .esn = 1 }, }, .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA }, { /* IPsec Inline Crypto ESP Transport Ingress */ .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, .protocol = RTE_SECURITY_PROTOCOL_IPSEC, .ipsec = { .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT, .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS, .options = { .udp_encap = 1, .stats = 1, .esn = 1 } }, .ol_flags = 0 }, { .action = RTE_SECURITY_ACTION_TYPE_NONE } }; /** * Update the security capabilities struct with the runtime discovered * crypto capabilities, except for last element of the array which is * the null termination */ for (i = 0; i < ((sizeof(iavf_security_capabilities) / sizeof(iavf_security_capabilities[0])) - 1); i++) { iavf_security_capabilities[i].crypto_capabilities = iavf_sctx->crypto_capabilities; } return iavf_security_capabilities; } static struct rte_security_ops iavf_ipsec_crypto_ops = { .session_get_size = iavf_ipsec_crypto_session_size_get, .session_create = iavf_ipsec_crypto_session_create, .session_update = iavf_ipsec_crypto_session_update, .session_stats_get = iavf_ipsec_crypto_session_stats_get, .session_destroy = iavf_ipsec_crypto_session_destroy, .set_pkt_metadata = iavf_ipsec_crypto_pkt_metadata_set, .get_userdata = NULL, .capabilities_get = iavf_ipsec_crypto_capabilities_get, }; int iavf_security_ctx_create(struct iavf_adapter *adapter) { struct rte_security_ctx *sctx; sctx = rte_malloc("security_ctx", sizeof(struct rte_security_ctx), 0); if (sctx == NULL) return -ENOMEM; sctx->device = adapter->vf.eth_dev; sctx->ops = &iavf_ipsec_crypto_ops; sctx->sess_cnt = 0; adapter->vf.eth_dev->security_ctx = sctx; if (adapter->security_ctx == NULL) { adapter->security_ctx = rte_malloc("iavf_security_ctx", sizeof(struct iavf_security_ctx), 0); if (adapter->security_ctx == NULL) return -ENOMEM; } return 0; } int iavf_security_init(struct iavf_adapter *adapter) { struct iavf_security_ctx *iavf_sctx = adapter->security_ctx; struct rte_mbuf_dynfield pkt_md_dynfield = { .name = "iavf_ipsec_crypto_pkt_metadata", .size = sizeof(struct iavf_ipsec_crypto_pkt_metadata), .align = __alignof__(struct iavf_ipsec_crypto_pkt_metadata) }; struct virtchnl_ipsec_cap capabilities; int rc; iavf_sctx->adapter = adapter; iavf_sctx->pkt_md_offset = rte_mbuf_dynfield_register(&pkt_md_dynfield); if (iavf_sctx->pkt_md_offset < 0) return iavf_sctx->pkt_md_offset; /* Get device capabilities from Inline IPsec driver over PF-VF comms */ rc = iavf_ipsec_crypto_device_capabilities_get(adapter, &capabilities); if (rc) return rc; return iavf_ipsec_crypto_set_security_capabililites(iavf_sctx, &capabilities); } int iavf_security_get_pkt_md_offset(struct iavf_adapter *adapter) { struct iavf_security_ctx *iavf_sctx = adapter->security_ctx; return iavf_sctx->pkt_md_offset; } int iavf_security_ctx_destroy(struct iavf_adapter *adapter) { struct rte_security_ctx *sctx = adapter->vf.eth_dev->security_ctx; struct iavf_security_ctx *iavf_sctx = adapter->security_ctx; if (iavf_sctx == NULL) return -ENODEV; /* free and reset security data structures */ rte_free(iavf_sctx); rte_free(sctx); adapter->security_ctx = NULL; adapter->vf.eth_dev->security_ctx = NULL; return 0; } static int iavf_ipsec_crypto_status_get(struct iavf_adapter *adapter, struct virtchnl_ipsec_status *status) { /* Perform pf-vf comms */ struct inline_ipsec_msg *request = NULL, *response = NULL; size_t request_len, response_len; int rc; request_len = sizeof(struct inline_ipsec_msg); request = rte_malloc("iavf-device-status-request", request_len, 0); if (request == NULL) { rc = -ENOMEM; goto update_cleanup; } response_len = sizeof(struct inline_ipsec_msg) + sizeof(struct virtchnl_ipsec_cap); response = rte_malloc("iavf-device-status-response", response_len, 0); if (response == NULL) { rc = -ENOMEM; goto update_cleanup; } /* set msg header params */ request->ipsec_opcode = INLINE_IPSEC_OP_GET_STATUS; request->req_id = (uint16_t)0xDEADBEEF; /* send virtual channel request to add SA to hardware database */ rc = iavf_ipsec_crypto_request(adapter, (uint8_t *)request, request_len, (uint8_t *)response, response_len); if (rc) goto update_cleanup; /* verify response id */ if (response->ipsec_opcode != request->ipsec_opcode || response->req_id != request->req_id){ rc = -EFAULT; goto update_cleanup; } memcpy(status, response->ipsec_data.ipsec_status, sizeof(*status)); update_cleanup: rte_free(response); rte_free(request); return rc; } int iavf_ipsec_crypto_supported(struct iavf_adapter *adapter) { struct virtchnl_vf_resource *resources = adapter->vf.vf_res; int crypto_supported = false; /** Capability check for IPsec Crypto */ if (resources && (resources->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_INLINE_IPSEC_CRYPTO)) { struct virtchnl_ipsec_status status; int rc = iavf_ipsec_crypto_status_get(adapter, &status); if (rc == 0 && status.status == INLINE_IPSEC_STATUS_AVAILABLE) crypto_supported = true; } /* Clear the VF flag to return faster next call */ if (resources && !crypto_supported) resources->vf_cap_flags &= ~(VIRTCHNL_VF_OFFLOAD_INLINE_IPSEC_CRYPTO); return crypto_supported; } #define IAVF_IPSEC_INSET_ESP (\ IAVF_INSET_ESP_SPI) #define IAVF_IPSEC_INSET_AH (\ IAVF_INSET_AH_SPI) #define IAVF_IPSEC_INSET_IPV4_NATT_ESP (\ IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \ IAVF_INSET_ESP_SPI) #define IAVF_IPSEC_INSET_IPV6_NATT_ESP (\ IAVF_INSET_IPV6_SRC | IAVF_INSET_IPV6_DST | \ IAVF_INSET_ESP_SPI) enum iavf_ipsec_flow_pt_type { IAVF_PATTERN_ESP = 1, IAVF_PATTERN_AH, IAVF_PATTERN_UDP_ESP, }; enum iavf_ipsec_flow_pt_ip_ver { IAVF_PATTERN_IPV4 = 1, IAVF_PATTERN_IPV6, }; #define IAVF_PATTERN(t, ipt) ((void *)((t) | ((ipt) << 4))) #define IAVF_PATTERN_TYPE(pt) ((pt) & 0x0F) #define IAVF_PATTERN_IP_V(pt) ((pt) >> 4) static struct iavf_pattern_match_item iavf_ipsec_flow_pattern[] = { {iavf_pattern_eth_ipv4_esp, IAVF_IPSEC_INSET_ESP, IAVF_PATTERN(IAVF_PATTERN_ESP, IAVF_PATTERN_IPV4)}, {iavf_pattern_eth_ipv6_esp, IAVF_IPSEC_INSET_ESP, IAVF_PATTERN(IAVF_PATTERN_ESP, IAVF_PATTERN_IPV6)}, {iavf_pattern_eth_ipv4_ah, IAVF_IPSEC_INSET_AH, IAVF_PATTERN(IAVF_PATTERN_AH, IAVF_PATTERN_IPV4)}, {iavf_pattern_eth_ipv6_ah, IAVF_IPSEC_INSET_AH, IAVF_PATTERN(IAVF_PATTERN_AH, IAVF_PATTERN_IPV6)}, {iavf_pattern_eth_ipv4_udp_esp, IAVF_IPSEC_INSET_IPV4_NATT_ESP, IAVF_PATTERN(IAVF_PATTERN_UDP_ESP, IAVF_PATTERN_IPV4)}, {iavf_pattern_eth_ipv6_udp_esp, IAVF_IPSEC_INSET_IPV6_NATT_ESP, IAVF_PATTERN(IAVF_PATTERN_UDP_ESP, IAVF_PATTERN_IPV6)}, }; struct iavf_ipsec_flow_item { uint64_t id; uint8_t is_ipv4; uint32_t spi; struct rte_ether_hdr eth_hdr; union { struct rte_ipv4_hdr ipv4_hdr; struct rte_ipv6_hdr ipv6_hdr; }; struct rte_udp_hdr udp_hdr; uint8_t is_udp; }; static void parse_eth_item(const struct rte_flow_item_eth *item, struct rte_ether_hdr *eth) { memcpy(eth->src_addr.addr_bytes, item->src.addr_bytes, sizeof(eth->src_addr)); memcpy(eth->dst_addr.addr_bytes, item->dst.addr_bytes, sizeof(eth->dst_addr)); } static void parse_ipv4_item(const struct rte_flow_item_ipv4 *item, struct rte_ipv4_hdr *ipv4) { ipv4->src_addr = item->hdr.src_addr; ipv4->dst_addr = item->hdr.dst_addr; } static void parse_ipv6_item(const struct rte_flow_item_ipv6 *item, struct rte_ipv6_hdr *ipv6) { memcpy(ipv6->src_addr, item->hdr.src_addr, 16); memcpy(ipv6->dst_addr, item->hdr.dst_addr, 16); } static void parse_udp_item(const struct rte_flow_item_udp *item, struct rte_udp_hdr *udp) { udp->dst_port = item->hdr.dst_port; udp->src_port = item->hdr.src_port; } static int has_security_action(const struct rte_flow_action actions[], const void **session) { /* only {SECURITY; END} supported */ if (actions[0].type == RTE_FLOW_ACTION_TYPE_SECURITY && actions[1].type == RTE_FLOW_ACTION_TYPE_END) { *session = actions[0].conf; return true; } return false; } static struct iavf_ipsec_flow_item * iavf_ipsec_flow_item_parse(struct rte_eth_dev *ethdev, const struct rte_flow_item pattern[], const struct rte_flow_action actions[], uint32_t type) { const void *session; struct iavf_ipsec_flow_item *ipsec_flow = rte_malloc("security-flow-rule", sizeof(struct iavf_ipsec_flow_item), 0); enum iavf_ipsec_flow_pt_type p_type = IAVF_PATTERN_TYPE(type); enum iavf_ipsec_flow_pt_ip_ver p_ip_type = IAVF_PATTERN_IP_V(type); if (ipsec_flow == NULL) return NULL; ipsec_flow->is_ipv4 = (p_ip_type == IAVF_PATTERN_IPV4); if (pattern[0].spec) parse_eth_item((const struct rte_flow_item_eth *) pattern[0].spec, &ipsec_flow->eth_hdr); switch (p_type) { case IAVF_PATTERN_ESP: if (ipsec_flow->is_ipv4) { parse_ipv4_item((const struct rte_flow_item_ipv4 *) pattern[1].spec, &ipsec_flow->ipv4_hdr); } else { parse_ipv6_item((const struct rte_flow_item_ipv6 *) pattern[1].spec, &ipsec_flow->ipv6_hdr); } ipsec_flow->spi = ((const struct rte_flow_item_esp *) pattern[2].spec)->hdr.spi; break; case IAVF_PATTERN_AH: if (ipsec_flow->is_ipv4) { parse_ipv4_item((const struct rte_flow_item_ipv4 *) pattern[1].spec, &ipsec_flow->ipv4_hdr); } else { parse_ipv6_item((const struct rte_flow_item_ipv6 *) pattern[1].spec, &ipsec_flow->ipv6_hdr); } ipsec_flow->spi = ((const struct rte_flow_item_ah *) pattern[2].spec)->spi; break; case IAVF_PATTERN_UDP_ESP: if (ipsec_flow->is_ipv4) { parse_ipv4_item((const struct rte_flow_item_ipv4 *) pattern[1].spec, &ipsec_flow->ipv4_hdr); } else { parse_ipv6_item((const struct rte_flow_item_ipv6 *) pattern[1].spec, &ipsec_flow->ipv6_hdr); } parse_udp_item((const struct rte_flow_item_udp *) pattern[2].spec, &ipsec_flow->udp_hdr); ipsec_flow->is_udp = true; ipsec_flow->spi = ((const struct rte_flow_item_esp *) pattern[3].spec)->hdr.spi; break; default: goto flow_cleanup; } if (!has_security_action(actions, &session)) goto flow_cleanup; if (!iavf_ipsec_crypto_action_valid(ethdev, session, ipsec_flow->spi)) goto flow_cleanup; return ipsec_flow; flow_cleanup: rte_free(ipsec_flow); return NULL; } static struct iavf_flow_parser iavf_ipsec_flow_parser; static int iavf_ipsec_flow_init(struct iavf_adapter *ad) { struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); struct iavf_flow_parser *parser; if (!vf->vf_res) return -EINVAL; if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_INLINE_IPSEC_CRYPTO) parser = &iavf_ipsec_flow_parser; else return -ENOTSUP; return iavf_register_parser(parser, ad); } static void iavf_ipsec_flow_uninit(struct iavf_adapter *ad) { iavf_unregister_parser(&iavf_ipsec_flow_parser, ad); } static int iavf_ipsec_flow_create(struct iavf_adapter *ad, struct rte_flow *flow, void *meta, struct rte_flow_error *error) { struct iavf_ipsec_flow_item *ipsec_flow = meta; int flow_id = -1; if (!ipsec_flow) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "NULL rule."); return -rte_errno; } if (ipsec_flow->is_ipv4) { flow_id = iavf_ipsec_crypto_inbound_security_policy_add(ad, ipsec_flow->spi, 1, ipsec_flow->ipv4_hdr.dst_addr, NULL, 0, ipsec_flow->is_udp, ipsec_flow->udp_hdr.dst_port); } else { flow_id = iavf_ipsec_crypto_inbound_security_policy_add(ad, ipsec_flow->spi, 0, 0, ipsec_flow->ipv6_hdr.dst_addr, 0, ipsec_flow->is_udp, ipsec_flow->udp_hdr.dst_port); } if (flow_id < 1) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "Failed to add SA."); return -rte_errno; } ipsec_flow->id = flow_id; flow->rule = ipsec_flow; return 0; } static int iavf_ipsec_flow_destroy(struct iavf_adapter *ad, struct rte_flow *flow, struct rte_flow_error *error) { struct iavf_ipsec_flow_item *ipsec_flow = flow->rule; if (!ipsec_flow) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "NULL rule."); return -rte_errno; } iavf_ipsec_crypto_security_policy_delete(ad, ipsec_flow->is_ipv4, ipsec_flow->id); rte_free(ipsec_flow); return 0; } static struct iavf_flow_engine iavf_ipsec_flow_engine = { .init = iavf_ipsec_flow_init, .uninit = iavf_ipsec_flow_uninit, .create = iavf_ipsec_flow_create, .destroy = iavf_ipsec_flow_destroy, .type = IAVF_FLOW_ENGINE_IPSEC_CRYPTO, }; static int iavf_ipsec_flow_parse(struct iavf_adapter *ad, struct iavf_pattern_match_item *array, uint32_t array_len, const struct rte_flow_item pattern[], const struct rte_flow_action actions[], void **meta, struct rte_flow_error *error) { struct iavf_pattern_match_item *item = NULL; int ret = -1; item = iavf_search_pattern_match_item(pattern, array, array_len, error); if (item && item->meta) { uint32_t type = (uint64_t)(item->meta); struct iavf_ipsec_flow_item *fi = iavf_ipsec_flow_item_parse(ad->vf.eth_dev, pattern, actions, type); if (fi && meta) { *meta = fi; ret = 0; } } return ret; } static struct iavf_flow_parser iavf_ipsec_flow_parser = { .engine = &iavf_ipsec_flow_engine, .array = iavf_ipsec_flow_pattern, .array_len = RTE_DIM(iavf_ipsec_flow_pattern), .parse_pattern_action = iavf_ipsec_flow_parse, .stage = IAVF_FLOW_STAGE_IPSEC_CRYPTO, }; RTE_INIT(iavf_ipsec_flow_engine_register) { iavf_register_flow_engine(&iavf_ipsec_flow_engine); }