Add rawdev enqueue operation for gdtc devices. Signed-off-by: Yong Zhang <zhang.yong25@zte.com.cn> --- drivers/raw/gdtc/gdtc_rawdev.c | 220 +++++++++++++++++++++++++++++++++ drivers/raw/gdtc/gdtc_rawdev.h | 19 +++ 2 files changed, 239 insertions(+) diff --git a/drivers/raw/gdtc/gdtc_rawdev.c b/drivers/raw/gdtc/gdtc_rawdev.c index 8512bd8413..f58b034e58 100644 --- a/drivers/raw/gdtc/gdtc_rawdev.c +++ b/drivers/raw/gdtc/gdtc_rawdev.c @@ -43,10 +43,34 @@ /* Register offset */ #define ZXDH_GDMA_BASE_OFFSET 0x100000 #define ZXDH_GDMA_EXT_ADDR_OFFSET 0x218 +#define ZXDH_GDMA_SAR_LOW_OFFSET 0x200 +#define ZXDH_GDMA_DAR_LOW_OFFSET 0x204 +#define ZXDH_GDMA_SAR_HIGH_OFFSET 0x234 +#define ZXDH_GDMA_DAR_HIGH_OFFSET 0x238 +#define ZXDH_GDMA_XFERSIZE_OFFSET 0x208 #define ZXDH_GDMA_CONTROL_OFFSET 0x230 +#define ZXDH_GDMA_TC_STATUS_OFFSET 0x0 +#define ZXDH_GDMA_STATUS_CLEAN_OFFSET 0x80 +#define ZXDH_GDMA_LLI_L_OFFSET 0x21c +#define ZXDH_GDMA_LLI_H_OFFSET 0x220 +#define ZXDH_GDMA_CHAN_CONTINUE_OFFSET 0x224 #define ZXDH_GDMA_TC_CNT_OFFSET 0x23c #define ZXDH_GDMA_LLI_USER_OFFSET 0x228 +/* Control register */ +#define ZXDH_GDMA_CHAN_ENABLE 0x1 +#define ZXDH_GDMA_CHAN_DISABLE 0 +#define ZXDH_GDMA_SOFT_CHAN 0x2 +#define ZXDH_GDMA_TC_INTR_ENABLE 0x10 +#define ZXDH_GDMA_ALL_INTR_ENABLE 0x30 +#define ZXDH_GDMA_SBS_SHIFT 6 /* src burst size */ +#define ZXDH_GDMA_SBL_SHIFT 9 /* src burst length */ +#define ZXDH_GDMA_DBS_SHIFT 13 /* dest burst size */ +#define ZXDH_GDMA_BURST_SIZE_MIN 0x1 /* 1 byte */ +#define ZXDH_GDMA_BURST_SIZE_MEDIUM 0x4 /* 4 word */ +#define ZXDH_GDMA_BURST_SIZE_MAX 0x6 /* 16 word */ +#define ZXDH_GDMA_DEFAULT_BURST_LEN 0xf /* 16 beats */ +#define ZXDH_GDMA_TC_CNT_ENABLE (1 << 27) #define ZXDH_GDMA_CHAN_FORCE_CLOSE (1 << 31) /* TC count & Error interrupt status register */ @@ -58,9 +82,15 @@ #define ZXDH_GDMA_TC_CNT_CLEAN (1) #define ZXDH_GDMA_CHAN_SHIFT 0x80 +#define ZXDH_GDMA_LINK_END_NODE (1 << 30) +#define ZXDH_GDMA_CHAN_CONTINUE (1) + #define LOW32_MASK 0xffffffff #define LOW16_MASK 0xffff +#define IDX_TO_ADDR(addr, idx, t) \ + ((t)((uintptr_t)(addr) + (idx) * sizeof(struct zxdh_gdma_buff_desc))) + static int zxdh_gdma_queue_init(struct rte_rawdev *dev, uint16_t queue_id); static int zxdh_gdma_queue_free(struct rte_rawdev *dev, uint16_t queue_id); @@ -301,6 +331,194 @@ zxdh_gdma_rawdev_get_attr(struct rte_rawdev *dev, return 0; } + +static inline void +zxdh_gdma_control_cal(uint32_t *val, uint8_t tc_enable) +{ + *val = (ZXDH_GDMA_CHAN_ENABLE | + ZXDH_GDMA_SOFT_CHAN | + (ZXDH_GDMA_DEFAULT_BURST_LEN << ZXDH_GDMA_SBL_SHIFT) | + (ZXDH_GDMA_BURST_SIZE_MAX << ZXDH_GDMA_SBS_SHIFT) | + (ZXDH_GDMA_BURST_SIZE_MAX << ZXDH_GDMA_DBS_SHIFT)); + + if (tc_enable != 0) + *val |= ZXDH_GDMA_TC_CNT_ENABLE; +} + +static inline uint32_t +zxdh_gdma_user_get(struct zxdh_gdma_queue *queue, struct zxdh_gdma_job *job) +{ + uint32_t src_user = 0; + uint32_t dst_user = 0; + + if ((job->flags & ZXDH_GDMA_JOB_DIR_MASK) == 0) { + ZXDH_PMD_LOG(DEBUG, "job flags:0x%x default user:0x%x", + job->flags, queue->user); + return queue->user; + } else if ((job->flags & ZXDH_GDMA_JOB_DIR_TX) != 0) { + src_user = ZXDH_GDMA_ZF_USER; + dst_user = ((job->pf_id << ZXDH_GDMA_PF_NUM_SHIFT) | + ((job->ep_id + ZXDH_GDMA_EPID_OFFSET) << ZXDH_GDMA_EP_ID_SHIFT)); + + if (job->vf_id != 0) + dst_user |= (ZXDH_GDMA_VF_EN | + ((job->vf_id - 1) << ZXDH_GDMA_VF_NUM_SHIFT)); + } else { + dst_user = ZXDH_GDMA_ZF_USER; + src_user = ((job->pf_id << ZXDH_GDMA_PF_NUM_SHIFT) | + ((job->ep_id + ZXDH_GDMA_EPID_OFFSET) << ZXDH_GDMA_EP_ID_SHIFT)); + + if (job->vf_id != 0) + src_user |= (ZXDH_GDMA_VF_EN | + ((job->vf_id - 1) << ZXDH_GDMA_VF_NUM_SHIFT)); + } + ZXDH_PMD_LOG(DEBUG, "job flags:0x%x ep_id:%u, pf_id:%u, vf_id:%u, user:0x%x", + job->flags, job->ep_id, job->pf_id, job->vf_id, + (src_user & LOW16_MASK) | (dst_user << 16)); + + return (src_user & LOW16_MASK) | (dst_user << 16); +} + +static inline void +zxdh_gdma_fill_bd(struct zxdh_gdma_queue *queue, struct zxdh_gdma_job *job) +{ + struct zxdh_gdma_buff_desc *bd = NULL; + uint32_t val = 0; + uint64_t next_bd_addr = 0; + uint16_t avail_idx = 0; + + avail_idx = queue->ring.avail_idx; + bd = &(queue->ring.desc[avail_idx]); + memset(bd, 0, sizeof(struct zxdh_gdma_buff_desc)); + + /* data bd */ + if (job != NULL) { + zxdh_gdma_control_cal(&val, 1); + next_bd_addr = IDX_TO_ADDR(queue->ring.ring_mem, + (avail_idx + 1) % ZXDH_GDMA_RING_SIZE, + uint64_t); + bd->SrcAddr_L = job->src & LOW32_MASK; + bd->DstAddr_L = job->dest & LOW32_MASK; + bd->SrcAddr_H = (job->src >> 32) & LOW32_MASK; + bd->DstAddr_H = (job->dest >> 32) & LOW32_MASK; + bd->Xpara = job->len; + bd->ExtAddr = zxdh_gdma_user_get(queue, job); + bd->LLI_Addr_L = (next_bd_addr >> 6) & LOW32_MASK; + bd->LLI_Addr_H = next_bd_addr >> 38; + bd->LLI_User = ZXDH_GDMA_ZF_USER; + bd->Control = val; + } else { + zxdh_gdma_control_cal(&val, 0); + next_bd_addr = IDX_TO_ADDR(queue->ring.ring_mem, avail_idx, uint64_t); + bd->ExtAddr = queue->user; + bd->LLI_User = ZXDH_GDMA_ZF_USER; + bd->Control = val; + bd->LLI_Addr_L = (next_bd_addr >> 6) & LOW32_MASK; + bd->LLI_Addr_H = (next_bd_addr >> 38) | ZXDH_GDMA_LINK_END_NODE; + if (queue->flag != 0) { + bd = IDX_TO_ADDR(queue->ring.desc, + queue->ring.last_avail_idx, + struct zxdh_gdma_buff_desc*); + next_bd_addr = IDX_TO_ADDR(queue->ring.ring_mem, + (queue->ring.last_avail_idx + 1) % ZXDH_GDMA_RING_SIZE, + uint64_t); + bd->LLI_Addr_L = (next_bd_addr >> 6) & LOW32_MASK; + bd->LLI_Addr_H = next_bd_addr >> 38; + rte_wmb(); + bd->LLI_Addr_H &= ~ZXDH_GDMA_LINK_END_NODE; + } + /* Record the index of empty bd for dynamic chaining */ + queue->ring.last_avail_idx = avail_idx; + } + + if (++avail_idx >= ZXDH_GDMA_RING_SIZE) + avail_idx -= ZXDH_GDMA_RING_SIZE; + + queue->ring.avail_idx = avail_idx; +} + +static int +zxdh_gdma_rawdev_enqueue_bufs(struct rte_rawdev *dev, + __rte_unused struct rte_rawdev_buf **buffers, + uint32_t count, + rte_rawdev_obj_t context) +{ + struct zxdh_gdma_rawdev *gdmadev = NULL; + struct zxdh_gdma_queue *queue = NULL; + struct zxdh_gdma_enqdeq *e_context = NULL; + struct zxdh_gdma_job *job = NULL; + uint16_t queue_id = 0; + uint32_t val = 0; + uint16_t i = 0; + uint16_t free_cnt = 0; + + if (dev == NULL) + return -EINVAL; + + if (unlikely((count < 1) || (context == NULL))) + return -EINVAL; + + gdmadev = zxdh_gdma_rawdev_get_priv(dev); + if (gdmadev->device_state == ZXDH_GDMA_DEV_STOPPED) { + ZXDH_PMD_LOG(ERR, "gdma dev is stop"); + return 0; + } + + e_context = (struct zxdh_gdma_enqdeq *)context; + queue_id = e_context->vq_id; + queue = zxdh_gdma_get_queue(dev, queue_id); + if ((queue == NULL) || (queue->enable == 0)) + return -EINVAL; + + free_cnt = queue->sw_ring.free_cnt; + if (free_cnt == 0) { + ZXDH_PMD_LOG(ERR, "queue %u is full, enq_idx:%u deq_idx:%u used_idx:%u", + queue_id, queue->sw_ring.enq_idx, + queue->sw_ring.deq_idx, queue->sw_ring.used_idx); + return 0; + } else if (free_cnt < count) { + ZXDH_PMD_LOG(DEBUG, "job num %u > free_cnt, change to %u", count, free_cnt); + count = free_cnt; + } + + rte_spinlock_lock(&queue->enqueue_lock); + + /* Build bd list, the last bd is empty bd */ + for (i = 0; i < count; i++) { + job = e_context->job[i]; + zxdh_gdma_fill_bd(queue, job); + } + zxdh_gdma_fill_bd(queue, NULL); + + if (unlikely(queue->flag == 0)) { + zxdh_gdma_write_reg(dev, queue_id, ZXDH_GDMA_LLI_L_OFFSET, + (queue->ring.ring_mem >> 6) & LOW32_MASK); + zxdh_gdma_write_reg(dev, queue_id, ZXDH_GDMA_LLI_H_OFFSET, + queue->ring.ring_mem >> 38); + /* Start hardware handling */ + zxdh_gdma_write_reg(dev, queue_id, ZXDH_GDMA_XFERSIZE_OFFSET, 0); + zxdh_gdma_control_cal(&val, 0); + zxdh_gdma_write_reg(dev, queue_id, ZXDH_GDMA_CONTROL_OFFSET, val); + queue->flag = 1; + } else { + val = ZXDH_GDMA_CHAN_CONTINUE; + zxdh_gdma_write_reg(dev, queue->vq_id, ZXDH_GDMA_CHAN_CONTINUE_OFFSET, val); + } + + /* job enqueue */ + for (i = 0; i < count; i++) { + queue->sw_ring.job[queue->sw_ring.enq_idx] = e_context->job[i]; + if (++queue->sw_ring.enq_idx >= queue->queue_size) + queue->sw_ring.enq_idx -= queue->queue_size; + + free_cnt--; + } + queue->sw_ring.free_cnt = free_cnt; + queue->sw_ring.pend_cnt += count; + rte_spinlock_unlock(&queue->enqueue_lock); + + return count; +} static const struct rte_rawdev_ops zxdh_gdma_rawdev_ops = { .dev_info_get = zxdh_gdma_rawdev_info_get, .dev_configure = zxdh_gdma_rawdev_configure, @@ -313,6 +531,8 @@ static const struct rte_rawdev_ops zxdh_gdma_rawdev_ops = { .queue_release = zxdh_gdma_rawdev_queue_release, .attr_get = zxdh_gdma_rawdev_get_attr, + + .enqueue_bufs = zxdh_gdma_rawdev_enqueue_bufs, }; static int diff --git a/drivers/raw/gdtc/gdtc_rawdev.h b/drivers/raw/gdtc/gdtc_rawdev.h index 92b35fcf14..12ce8a04c7 100644 --- a/drivers/raw/gdtc/gdtc_rawdev.h +++ b/drivers/raw/gdtc/gdtc_rawdev.h @@ -24,6 +24,20 @@ extern int zxdh_gdma_rawdev_logtype; #define ZXDH_GDMA_QUEUE_SIZE 16384 #define ZXDH_GDMA_RING_SIZE 32768 +/* States if the source addresses is physical. */ +#define ZXDH_GDMA_JOB_SRC_PHY (1UL) + +/* States if the destination addresses is physical. */ +#define ZXDH_GDMA_JOB_DEST_PHY (1UL << 1) + +/* ZF->HOST */ +#define ZXDH_GDMA_JOB_DIR_TX (1UL << 2) + +/* HOST->ZF */ +#define ZXDH_GDMA_JOB_DIR_RX (1UL << 3) + +#define ZXDH_GDMA_JOB_DIR_MASK (ZXDH_GDMA_JOB_DIR_TX | ZXDH_GDMA_JOB_DIR_RX) + enum zxdh_gdma_device_state { ZXDH_GDMA_DEV_RUNNING, ZXDH_GDMA_DEV_STOPPED @@ -100,6 +114,11 @@ struct zxdh_gdma_rawdev { struct zxdh_gdma_queue vqs[ZXDH_GDMA_TOTAL_CHAN_NUM]; }; +struct zxdh_gdma_enqdeq { + uint16_t vq_id; + struct zxdh_gdma_job **job; +}; + struct zxdh_gdma_config { uint16_t max_hw_queues_per_core; uint16_t max_vqs; -- 2.43.0