<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<style type="text/css" style="display:none;"> P {margin-top:0;margin-bottom:0;} </style>
</head>
<body dir="ltr">
<div style="font-family: "IntelOne Text"; font-size: 10pt; color: rgb(0, 0, 0);" class="elementToProof">
Series-acked-by: Kai Ji <kai.ji@intel.com></div>
<div style="font-family: "IntelOne Text"; font-size: 10pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div id="appendonsend"></div>
<hr style="display:inline-block;width:98%" tabindex="-1">
<div id="divRplyFwdMsg" dir="ltr"><font face="Calibri, sans-serif" style="font-size:11pt" color="#000000"><b>From:</b> Nicolau, Radu <radu.nicolau@intel.com><br>
<b>Sent:</b> 11 September 2025 14:14<br>
<b>To:</b> dev@dpdk.org <dev@dpdk.org><br>
<b>Cc:</b> Nicolau, Radu <radu.nicolau@intel.com>; Ji, Kai <kai.ji@intel.com>; De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com><br>
<b>Subject:</b> [PATCH v2 1/3] crypto/ipsec_mb: improve CPU code path</font>
<div> </div>
</div>
<div class="BodyFragment"><font size="2"><span style="font-size:11pt;">
<div class="PlainText">Improve CPU code path with support for OOP and various small fixes.<br>
<br>
Signed-off-by: Radu Nicolau <radu.nicolau@intel.com><br>
---<br>
 drivers/crypto/ipsec_mb/pmd_aesni_mb.c | 124 +++++++++++++++++--------<br>
 1 file changed, 86 insertions(+), 38 deletions(-)<br>
<br>
diff --git a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c<br>
index a6c3f09b6f..356e6cd4a0 100644<br>
--- a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c<br>
+++ b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c<br>
@@ -975,14 +975,24 @@ aesni_mb_set_docsis_sec_session_parameters(<br>
         return ret;<br>
 }<br>
 <br>
+static inline uint8_t *get_src_buf(struct rte_crypto_op *op)<br>
+{<br>
+       return op->sym->m_src ?<br>
+       rte_pktmbuf_mtod(op->sym->m_src, uint8_t *) : NULL;<br>
+}<br>
+static inline uint8_t *get_dst_buf(struct rte_crypto_op *op)<br>
+{<br>
+       return op->sym->m_dst ?<br>
+       rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *) : NULL;<br>
+}<br>
+<br>
 static inline uint64_t<br>
-auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,<br>
+auth_start_offset(uint8_t *p_src, uint8_t *p_dst,<br>
+               struct aesni_mb_session *session,<br>
                 uint32_t oop, const uint32_t auth_offset,<br>
                 const uint32_t cipher_offset, const uint32_t auth_length,<br>
                 const uint32_t cipher_length, uint8_t lb_sgl)<br>
 {<br>
-       struct rte_mbuf *m_src, *m_dst;<br>
-       uint8_t *p_src, *p_dst;<br>
         uintptr_t u_src, u_dst;<br>
         uint32_t cipher_end, auth_end;<br>
 <br>
@@ -990,11 +1000,6 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,<br>
         if (!oop || session->template_job.chain_order != IMB_ORDER_CIPHER_HASH || lb_sgl)<br>
                 return auth_offset;<br>
 <br>
-       m_src = op->sym->m_src;<br>
-       m_dst = op->sym->m_dst;<br>
-<br>
-       p_src = rte_pktmbuf_mtod(m_src, uint8_t *);<br>
-       p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);<br>
         u_src = (uintptr_t)p_src;<br>
         u_dst = (uintptr_t)p_dst + auth_offset;<br>
 <br>
@@ -1029,9 +1034,11 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,<br>
 <br>
 static inline void<br>
 set_cpu_mb_job_params(IMB_JOB *job, struct aesni_mb_session *session,<br>
-               union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,<br>
-               struct rte_crypto_va_iova_ptr *iv,<br>
-               struct rte_crypto_va_iova_ptr *aad, void *digest, void *udata)<br>
+               union rte_crypto_sym_ofs sofs, void *src_buf, void *dst_buf,<br>
+               uint32_t len, struct rte_crypto_va_iova_ptr *iv,<br>
+               struct rte_crypto_va_iova_ptr *aad,<br>
+               void **digest, struct rte_crypto_va_iova_ptr *vec_digest,<br>
+               void *udata)<br>
 {<br>
         memcpy(job, &session->template_job, sizeof(IMB_JOB));<br>
 <br>
@@ -1065,10 +1072,7 @@ set_cpu_mb_job_params(IMB_JOB *job, struct aesni_mb_session *session,<br>
 #endif<br>
 <br>
         default:<br>
-               job->u.HMAC._hashed_auth_key_xor_ipad =<br>
-                               session->auth.pads.inner;<br>
-               job->u.HMAC._hashed_auth_key_xor_opad =<br>
-                               session->auth.pads.outer;<br>
+               break;<br>
         }<br>
 <br>
         /*<br>
@@ -1076,20 +1080,48 @@ set_cpu_mb_job_params(IMB_JOB *job, struct aesni_mb_session *session,<br>
          * digest length as specified in the relevant IPsec RFCs<br>
          */<br>
 <br>
-       /* Set digest location and length */<br>
-       job->auth_tag_output = digest;<br>
-<br>
         /* Data Parameters */<br>
-       job->src = buf;<br>
-       job->dst = (uint8_t *)buf + sofs.ofs.cipher.head;<br>
+       job->src = src_buf;<br>
+       if (dst_buf)<br>
+               job->dst = (uint8_t *)dst_buf + sofs.ofs.cipher.head;<br>
+       else<br>
+               job->dst = (uint8_t *)src_buf + sofs.ofs.cipher.head;<br>
         job->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;<br>
-       job->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;<br>
         job->msg_len_to_hash_in_bytes = len - sofs.ofs.auth.head -<br>
                 sofs.ofs.auth.tail;<br>
         job->msg_len_to_cipher_in_bytes = len - sofs.ofs.cipher.head -<br>
                 sofs.ofs.cipher.tail;<br>
+       job->hash_start_src_offset_in_bytes = auth_start_offset(<br>
+                               src_buf, dst_buf,<br>
+                               session, dst_buf != NULL,<br>
+                               sofs.ofs.auth.head,<br>
+                               sofs.ofs.cipher.head,<br>
+                               job->msg_len_to_hash_in_bytes,<br>
+                               job->msg_len_to_cipher_in_bytes, 0);<br>
 <br>
         job->user_data = udata;<br>
+<br>
+       /** Check if conditions are met for digest-appended operations */<br>
+       if (job->cipher_mode != IMB_CIPHER_NULL && job->chain_order == IMB_ORDER_HASH_CIPHER) {<br>
+               uintptr_t end_cipher, start_cipher;<br>
+               if (dst_buf == NULL) {<br>
+                       end_cipher = (uintptr_t)src_buf + job->msg_len_to_cipher_in_bytes;<br>
+                       start_cipher = (uintptr_t)src_buf;<br>
+               } else {<br>
+                       end_cipher = (uintptr_t)dst_buf + job->msg_len_to_cipher_in_bytes;<br>
+                       start_cipher = (uintptr_t)dst_buf;<br>
+               }<br>
+               if (start_cipher < (uintptr_t)vec_digest->va &&<br>
+                       (uintptr_t)vec_digest->va < end_cipher)<br>
+                       *digest = (void *)((uintptr_t)src_buf + job->msg_len_to_hash_in_bytes);<br>
+       }<br>
+       job->auth_tag_output = *digest;<br>
+<br>
+       if (job->cipher_mode == IMB_CIPHER_NULL && dst_buf) {<br>
+               memcpy(job->dst + job->cipher_start_src_offset_in_bytes,<br>
+                       job->src + job->cipher_start_src_offset_in_bytes,<br>
+                       job->msg_len_to_cipher_in_bytes);<br>
+       }<br>
 }<br>
 <br>
 static int<br>
@@ -1693,7 +1725,8 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,<br>
                 auth_len_in_bytes = op->sym->auth.data.length >> 3;<br>
                 ciph_len_in_bytes = op->sym->cipher.data.length >> 3;<br>
 <br>
-               job->hash_start_src_offset_in_bytes = auth_start_offset(op,<br>
+               job->hash_start_src_offset_in_bytes = auth_start_offset(<br>
+                               get_src_buf(op), get_dst_buf(op),<br>
                                 session, oop, auth_off_in_bytes,<br>
                                 ciph_off_in_bytes, auth_len_in_bytes,<br>
                                 ciph_len_in_bytes, lb_sgl);<br>
@@ -1710,7 +1743,8 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,<br>
                 auth_len_in_bytes = op->sym->auth.data.length >> 3;<br>
                 ciph_len_in_bytes = op->sym->cipher.data.length >> 3;<br>
 <br>
-               job->hash_start_src_offset_in_bytes = auth_start_offset(op,<br>
+               job->hash_start_src_offset_in_bytes = auth_start_offset(<br>
+                               get_src_buf(op), get_dst_buf(op),<br>
                                 session, oop, auth_off_in_bytes,<br>
                                 ciph_off_in_bytes, auth_len_in_bytes,<br>
                                 ciph_len_in_bytes, lb_sgl);<br>
@@ -1735,7 +1769,8 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,<br>
 #endif<br>
 <br>
         default:<br>
-               job->hash_start_src_offset_in_bytes = auth_start_offset(op,<br>
+               job->hash_start_src_offset_in_bytes = auth_start_offset(<br>
+                               get_src_buf(op), get_dst_buf(op),<br>
                                 session, oop, op->sym->auth.data.offset,<br>
                                 op->sym->cipher.data.offset,<br>
                                 op->sym->auth.data.length,<br>
@@ -2261,12 +2296,16 @@ aesni_mb_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,<br>
 }<br>
 <br>
 static inline int<br>
-check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)<br>
+check_crypto_sgl(union rte_crypto_sym_ofs so,<br>
+       const struct rte_crypto_sym_vec *vec, uint32_t idx)<br>
 {<br>
+       const struct rte_crypto_sgl *sgl_src = vec->src_sgl + idx;<br>
+       const struct rte_crypto_sgl *sgl_dst = vec->dest_sgl ?<br>
+               vec->dest_sgl + idx : NULL;<br>
         /* no multi-seg support with current AESNI-MB PMD */<br>
-       if (sgl->num != 1)<br>
+       if (sgl_src->num != 1 || (sgl_dst && sgl_dst->num != 1))<br>
                 return -ENOTSUP;<br>
-       else if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)<br>
+       else if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl_src->vec[0].len)<br>
                 return -EINVAL;<br>
         return 0;<br>
 }<br>
@@ -2275,7 +2314,11 @@ static inline IMB_JOB *<br>
 submit_sync_job(IMB_MGR *mb_mgr)<br>
 {<br>
 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG<br>
-       return IMB_SUBMIT_JOB(mb_mgr);<br>
+       IMB_JOB *job = IMB_SUBMIT_JOB(mb_mgr);<br>
+       int err = imb_get_errno(mb_mgr);<br>
+       if (err)<br>
+               IPSEC_MB_LOG(ERR, "%s", imb_get_strerror(err));<br>
+       return job;<br>
 #else<br>
         return IMB_SUBMIT_JOB_NOCHECK(mb_mgr);<br>
 #endif<br>
@@ -2283,7 +2326,7 @@ submit_sync_job(IMB_MGR *mb_mgr)<br>
 <br>
 static inline uint32_t<br>
 generate_sync_dgst(struct rte_crypto_sym_vec *vec,<br>
-       const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)<br>
+       uint8_t *dgst[], uint32_t len)<br>
 {<br>
         uint32_t i, k;<br>
 <br>
@@ -2299,7 +2342,7 @@ generate_sync_dgst(struct rte_crypto_sym_vec *vec,<br>
 <br>
 static inline uint32_t<br>
 verify_sync_dgst(struct rte_crypto_sym_vec *vec,<br>
-       const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)<br>
+       uint8_t *dgst[], uint32_t len)<br>
 {<br>
         uint32_t i, k;<br>
 <br>
@@ -2322,11 +2365,12 @@ aesni_mb_process_bulk(struct rte_cryptodev *dev __rte_unused,<br>
 {<br>
         int32_t ret;<br>
         uint32_t i, j, k, len;<br>
-       void *buf;<br>
+       void *src_buf, *dst_buf = NULL;<br>
         IMB_JOB *job;<br>
         IMB_MGR *mb_mgr;<br>
         struct aesni_mb_session *s = CRYPTODEV_GET_SYM_SESS_PRIV(sess);<br>
         uint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];<br>
+       uint8_t *tmp_dgst_ptr[vec->num];<br>
 <br>
         /* get per-thread MB MGR, create one if needed */<br>
         mb_mgr = get_per_thread_mb_mgr();<br>
@@ -2334,13 +2378,15 @@ aesni_mb_process_bulk(struct rte_cryptodev *dev __rte_unused,<br>
                 return 0;<br>
 <br>
         for (i = 0, j = 0, k = 0; i != vec->num; i++) {<br>
-               ret = check_crypto_sgl(sofs, vec->src_sgl + i);<br>
+               ret = check_crypto_sgl(sofs, vec, i);<br>
                 if (ret != 0) {<br>
                         vec->status[i] = ret;<br>
                         continue;<br>
                 }<br>
 <br>
-               buf = vec->src_sgl[i].vec[0].base;<br>
+               src_buf = vec->src_sgl[i].vec[0].base;<br>
+               if (vec->dest_sgl != NULL && vec->dest_sgl[i].vec[0].base != NULL)<br>
+                       dst_buf = vec->dest_sgl[i].vec[0].base;<br>
                 len = vec->src_sgl[i].vec[0].len;<br>
 <br>
                 job = IMB_GET_NEXT_JOB(mb_mgr);<br>
@@ -2351,8 +2397,10 @@ aesni_mb_process_bulk(struct rte_cryptodev *dev __rte_unused,<br>
                 }<br>
 <br>
                 /* Submit job for processing */<br>
-               set_cpu_mb_job_params(job, s, sofs, buf, len, &vec->iv[i],<br>
-                       &vec->aad[i], tmp_dgst[i], &vec->status[i]);<br>
+               tmp_dgst_ptr[i] = tmp_dgst[i];<br>
+               set_cpu_mb_job_params(job, s, sofs, src_buf, dst_buf, len, &vec->iv[i],<br>
+                       &vec->aad[i], (void **)&tmp_dgst_ptr[i], &vec->digest[i],<br>
+                       &vec->status[i]);<br>
                 job = submit_sync_job(mb_mgr);<br>
                 j++;<br>
 <br>
@@ -2365,14 +2413,14 @@ aesni_mb_process_bulk(struct rte_cryptodev *dev __rte_unused,<br>
                 k += flush_mb_sync_mgr(mb_mgr);<br>
 <br>
         /* finish processing for successful jobs: check/update digest */<br>
-       if (k != 0) {<br>
+       if (k != 0 && s->template_job.hash_alg != IMB_AUTH_NULL) {<br>
                 if (s->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)<br>
                         k = verify_sync_dgst(vec,<br>
-                               (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,<br>
+                               tmp_dgst_ptr,<br>
                                 s->auth.req_digest_len);<br>
                 else<br>
                         k = generate_sync_dgst(vec,<br>
-                               (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,<br>
+                               tmp_dgst_ptr,<br>
                                 s->auth.req_digest_len);<br>
         }<br>
 <br>
-- <br>
2.50.1<br>
<br>
</div>
</span></font></div>
</body>
</html>