[dpdk-dev] [PATCH v2 09/46] net/liquidio: add mbox APIs for PF VF communication

[Shijith Thotton]

Signed-off-by: Shijith Thotton <shijith.thotton at caviumnetworks.com>
Signed-off-by: Jerin Jacob <jerin.jacob at caviumnetworks.com>
Signed-off-by: Derek Chickles <derek.chickles at caviumnetworks.com>
Signed-off-by: Venkat Koppula <venkat.koppula at caviumnetworks.com>
Signed-off-by: Srisivasubramanian S <ssrinivasan at caviumnetworks.com>
Signed-off-by: Mallesham Jatharakonda <mjatharakonda at oneconvergence.com>
---
 drivers/net/liquidio/Makefile        |   1 +
 drivers/net/liquidio/base/lio_mbox.c | 275 +++++++++++++++++++++++++++++++++++
 drivers/net/liquidio/base/lio_mbox.h | 129 ++++++++++++++++
 drivers/net/liquidio/lio_struct.h    |   3 +
 4 files changed, 408 insertions(+)
 create mode 100644 drivers/net/liquidio/base/lio_mbox.c
 create mode 100644 drivers/net/liquidio/base/lio_mbox.h

diff --git a/drivers/net/liquidio/Makefile b/drivers/net/liquidio/Makefile
index 8880a10..451f49d 100644
--- a/drivers/net/liquidio/Makefile
+++ b/drivers/net/liquidio/Makefile
@@ -52,6 +52,7 @@ VPATH += $(RTE_SDK)/drivers/net/liquidio/base
 #
 SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_23xx_vf.c
+SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_mbox.c
 
 # this lib depends upon:
 DEPDIRS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lib/librte_eal lib/librte_ether
diff --git a/drivers/net/liquidio/base/lio_mbox.c b/drivers/net/liquidio/base/lio_mbox.c
new file mode 100644
index 0000000..b4abc62
--- /dev/null
+++ b/drivers/net/liquidio/base/lio_mbox.c
@@ -0,0 +1,275 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Cavium, Inc.. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium, Inc. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_ethdev.h>
+#include <rte_cycles.h>
+
+#include "lio_logs.h"
+#include "lio_struct.h"
+#include "lio_mbox.h"
+
+/**
+ * lio_mbox_read:
+ * @mbox: Pointer mailbox
+ *
+ * Reads the 8-bytes of data from the mbox register
+ * Writes back the acknowledgment indicating completion of read
+ */
+int
+lio_mbox_read(struct lio_mbox *mbox)
+{
+	union lio_mbox_message msg;
+	int ret = 0;
+
+	msg.mbox_msg64 = rte_read64(mbox->mbox_read_reg);
+
+	if ((msg.mbox_msg64 == LIO_PFVFACK) || (msg.mbox_msg64 == LIO_PFVFSIG))
+		return 0;
+
+	if (mbox->state & LIO_MBOX_STATE_REQ_RECEIVING) {
+		mbox->mbox_req.data[mbox->mbox_req.recv_len - 1] =
+					msg.mbox_msg64;
+		mbox->mbox_req.recv_len++;
+	} else {
+		if (mbox->state & LIO_MBOX_STATE_RES_RECEIVING) {
+			mbox->mbox_resp.data[mbox->mbox_resp.recv_len - 1] =
+					msg.mbox_msg64;
+			mbox->mbox_resp.recv_len++;
+		} else {
+			if ((mbox->state & LIO_MBOX_STATE_IDLE) &&
+					(msg.s.type == LIO_MBOX_REQUEST)) {
+				mbox->state &= ~LIO_MBOX_STATE_IDLE;
+				mbox->state |= LIO_MBOX_STATE_REQ_RECEIVING;
+				mbox->mbox_req.msg.mbox_msg64 = msg.mbox_msg64;
+				mbox->mbox_req.q_no = mbox->q_no;
+				mbox->mbox_req.recv_len = 1;
+			} else {
+				if ((mbox->state &
+				     LIO_MBOX_STATE_RES_PENDING) &&
+				    (msg.s.type == LIO_MBOX_RESPONSE)) {
+					mbox->state &=
+						~LIO_MBOX_STATE_RES_PENDING;
+					mbox->state |=
+						LIO_MBOX_STATE_RES_RECEIVING;
+					mbox->mbox_resp.msg.mbox_msg64 =
+								msg.mbox_msg64;
+					mbox->mbox_resp.q_no = mbox->q_no;
+					mbox->mbox_resp.recv_len = 1;
+				} else {
+					rte_write64(LIO_PFVFERR,
+						    mbox->mbox_read_reg);
+					mbox->state |= LIO_MBOX_STATE_ERROR;
+					return -1;
+				}
+			}
+		}
+	}
+
+	if (mbox->state & LIO_MBOX_STATE_REQ_RECEIVING) {
+		if (mbox->mbox_req.recv_len < msg.s.len) {
+			ret = 0;
+		} else {
+			mbox->state &= ~LIO_MBOX_STATE_REQ_RECEIVING;
+			mbox->state |= LIO_MBOX_STATE_REQ_RECEIVED;
+			ret = 1;
+		}
+	} else {
+		if (mbox->state & LIO_MBOX_STATE_RES_RECEIVING) {
+			if (mbox->mbox_resp.recv_len < msg.s.len) {
+				ret = 0;
+			} else {
+				mbox->state &= ~LIO_MBOX_STATE_RES_RECEIVING;
+				mbox->state |= LIO_MBOX_STATE_RES_RECEIVED;
+				ret = 1;
+			}
+		} else {
+			RTE_ASSERT(0);
+		}
+	}
+
+	rte_write64(LIO_PFVFACK, mbox->mbox_read_reg);
+
+	return ret;
+}
+
+/**
+ * lio_mbox_write:
+ * @lio_dev: Pointer lio device
+ * @mbox_cmd: Cmd to send to mailbox.
+ *
+ * Populates the queue specific mbox structure
+ * with cmd information.
+ * Write the cmd to mbox register
+ */
+int
+lio_mbox_write(struct lio_device *lio_dev,
+	       struct lio_mbox_cmd *mbox_cmd)
+{
+	struct lio_mbox *mbox = lio_dev->mbox[mbox_cmd->q_no];
+	uint32_t count, i, ret = LIO_MBOX_STATUS_SUCCESS;
+
+	if ((mbox_cmd->msg.s.type == LIO_MBOX_RESPONSE) &&
+			!(mbox->state & LIO_MBOX_STATE_REQ_RECEIVED))
+		return LIO_MBOX_STATUS_FAILED;
+
+	if ((mbox_cmd->msg.s.type == LIO_MBOX_REQUEST) &&
+			!(mbox->state & LIO_MBOX_STATE_IDLE))
+		return LIO_MBOX_STATUS_BUSY;
+
+	if (mbox_cmd->msg.s.type == LIO_MBOX_REQUEST) {
+		rte_memcpy(&mbox->mbox_resp, mbox_cmd,
+			   sizeof(struct lio_mbox_cmd));
+		mbox->state = LIO_MBOX_STATE_RES_PENDING;
+	}
+
+	count = 0;
+
+	while (rte_read64(mbox->mbox_write_reg) != LIO_PFVFSIG) {
+		rte_delay_ms(1);
+		if (count++ == 1000) {
+			ret = LIO_MBOX_STATUS_FAILED;
+			break;
+		}
+	}
+
+	if (ret == LIO_MBOX_STATUS_SUCCESS) {
+		rte_write64(mbox_cmd->msg.mbox_msg64, mbox->mbox_write_reg);
+		for (i = 0; i < (uint32_t)(mbox_cmd->msg.s.len - 1); i++) {
+			count = 0;
+			while (rte_read64(mbox->mbox_write_reg) !=
+					LIO_PFVFACK) {
+				rte_delay_ms(1);
+				if (count++ == 1000) {
+					ret = LIO_MBOX_STATUS_FAILED;
+					break;
+				}
+			}
+			rte_write64(mbox_cmd->data[i], mbox->mbox_write_reg);
+		}
+	}
+
+	if (mbox_cmd->msg.s.type == LIO_MBOX_RESPONSE) {
+		mbox->state = LIO_MBOX_STATE_IDLE;
+		rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg);
+	} else {
+		if ((!mbox_cmd->msg.s.resp_needed) ||
+				(ret == LIO_MBOX_STATUS_FAILED)) {
+			mbox->state &= ~LIO_MBOX_STATE_RES_PENDING;
+			if (!(mbox->state & (LIO_MBOX_STATE_REQ_RECEIVING |
+					     LIO_MBOX_STATE_REQ_RECEIVED)))
+				mbox->state = LIO_MBOX_STATE_IDLE;
+		}
+	}
+
+	return ret;
+}
+
+/**
+ * lio_mbox_process_cmd:
+ * @mbox: Pointer mailbox
+ * @mbox_cmd: Pointer to command received
+ *
+ * Process the cmd received in mbox
+ */
+static int
+lio_mbox_process_cmd(struct lio_mbox *mbox,
+		     struct lio_mbox_cmd *mbox_cmd)
+{
+	struct lio_device *lio_dev = mbox->lio_dev;
+
+	if (mbox_cmd->msg.s.cmd == LIO_CORES_CRASHED)
+		lio_dev_err(lio_dev, "Octeon core(s) crashed or got stuck!\n");
+
+	return 0;
+}
+
+/**
+ * Process the received mbox message.
+ */
+int
+lio_mbox_process_message(struct lio_mbox *mbox)
+{
+	struct lio_mbox_cmd mbox_cmd;
+
+	if (mbox->state & LIO_MBOX_STATE_ERROR) {
+		if (mbox->state & (LIO_MBOX_STATE_RES_PENDING |
+				   LIO_MBOX_STATE_RES_RECEIVING)) {
+			rte_memcpy(&mbox_cmd, &mbox->mbox_resp,
+				   sizeof(struct lio_mbox_cmd));
+			mbox->state = LIO_MBOX_STATE_IDLE;
+			rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg);
+			mbox_cmd.recv_status = 1;
+			if (mbox_cmd.fn)
+				mbox_cmd.fn(mbox->lio_dev, &mbox_cmd,
+					    mbox_cmd.fn_arg);
+
+			return 0;
+		}
+
+		mbox->state = LIO_MBOX_STATE_IDLE;
+		rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg);
+
+		return 0;
+	}
+
+	if (mbox->state & LIO_MBOX_STATE_RES_RECEIVED) {
+		rte_memcpy(&mbox_cmd, &mbox->mbox_resp,
+			   sizeof(struct lio_mbox_cmd));
+		mbox->state = LIO_MBOX_STATE_IDLE;
+		rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg);
+		mbox_cmd.recv_status = 0;
+		if (mbox_cmd.fn)
+			mbox_cmd.fn(mbox->lio_dev, &mbox_cmd, mbox_cmd.fn_arg);
+
+		return 0;
+	}
+
+	if (mbox->state & LIO_MBOX_STATE_REQ_RECEIVED) {
+		rte_memcpy(&mbox_cmd, &mbox->mbox_req,
+			   sizeof(struct lio_mbox_cmd));
+		if (!mbox_cmd.msg.s.resp_needed) {
+			mbox->state &= ~LIO_MBOX_STATE_REQ_RECEIVED;
+			if (!(mbox->state & LIO_MBOX_STATE_RES_PENDING))
+				mbox->state = LIO_MBOX_STATE_IDLE;
+			rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg);
+		}
+
+		lio_mbox_process_cmd(mbox, &mbox_cmd);
+
+		return 0;
+	}
+
+	RTE_ASSERT(0);
+
+	return 0;
+}
diff --git a/drivers/net/liquidio/base/lio_mbox.h b/drivers/net/liquidio/base/lio_mbox.h
new file mode 100644
index 0000000..28c9e1a
--- /dev/null
+++ b/drivers/net/liquidio/base/lio_mbox.h
@@ -0,0 +1,129 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Cavium, Inc.. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium, Inc. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _LIO_MBOX_H_
+#define _LIO_MBOX_H_
+
+#include <stdint.h>
+
+#include <rte_spinlock.h>
+
+/* Macros for Mail Box Communication */
+
+#define LIO_MBOX_DATA_MAX			32
+
+#define LIO_CORES_CRASHED			0x3
+
+/* Macro for Read acknowledgment */
+#define LIO_PFVFACK				0xffffffffffffffff
+#define LIO_PFVFSIG				0x1122334455667788
+#define LIO_PFVFERR				0xDEADDEADDEADDEAD
+
+enum lio_mbox_cmd_status {
+	LIO_MBOX_STATUS_SUCCESS		= 0,
+	LIO_MBOX_STATUS_FAILED		= 1,
+	LIO_MBOX_STATUS_BUSY		= 2
+};
+
+enum lio_mbox_message_type {
+	LIO_MBOX_REQUEST	= 0,
+	LIO_MBOX_RESPONSE	= 1
+};
+
+union lio_mbox_message {
+	uint64_t mbox_msg64;
+	struct {
+		uint16_t type : 1;
+		uint16_t resp_needed : 1;
+		uint16_t cmd : 6;
+		uint16_t len : 8;
+		uint8_t params[6];
+	} s;
+};
+
+typedef void (*lio_mbox_callback)(void *, void *, void *);
+
+struct lio_mbox_cmd {
+	union lio_mbox_message msg;
+	uint64_t data[LIO_MBOX_DATA_MAX];
+	uint32_t q_no;
+	uint32_t recv_len;
+	uint32_t recv_status;
+	lio_mbox_callback fn;
+	void *fn_arg;
+};
+
+enum lio_mbox_state {
+	LIO_MBOX_STATE_IDLE		= 1,
+	LIO_MBOX_STATE_REQ_RECEIVING	= 2,
+	LIO_MBOX_STATE_REQ_RECEIVED	= 4,
+	LIO_MBOX_STATE_RES_PENDING	= 8,
+	LIO_MBOX_STATE_RES_RECEIVING	= 16,
+	LIO_MBOX_STATE_RES_RECEIVED	= 16,
+	LIO_MBOX_STATE_ERROR		= 32
+};
+
+struct lio_mbox {
+	/* A spinlock to protect access to this q_mbox. */
+	rte_spinlock_t lock;
+
+	struct lio_device *lio_dev;
+
+	uint32_t q_no;
+
+	enum lio_mbox_state state;
+
+	/* SLI_MAC_PF_MBOX_INT for PF, SLI_PKT_MBOX_INT for VF. */
+	void *mbox_int_reg;
+
+	/* SLI_PKT_PF_VF_MBOX_SIG(0) for PF,
+	 * SLI_PKT_PF_VF_MBOX_SIG(1) for VF.
+	 */
+	void *mbox_write_reg;
+
+	/* SLI_PKT_PF_VF_MBOX_SIG(1) for PF,
+	 * SLI_PKT_PF_VF_MBOX_SIG(0) for VF.
+	 */
+	void *mbox_read_reg;
+
+	struct lio_mbox_cmd mbox_req;
+
+	struct lio_mbox_cmd mbox_resp;
+
+};
+
+int lio_mbox_read(struct lio_mbox *mbox);
+int lio_mbox_write(struct lio_device *lio_dev,
+		   struct lio_mbox_cmd *mbox_cmd);
+int lio_mbox_process_message(struct lio_mbox *mbox);
+#endif	/* _LIO_MBOX_H_ */
diff --git a/drivers/net/liquidio/lio_struct.h b/drivers/net/liquidio/lio_struct.h
index 577ea49..0af4fe3 100644
--- a/drivers/net/liquidio/lio_struct.h
+++ b/drivers/net/liquidio/lio_struct.h
@@ -126,6 +126,9 @@ struct lio_device {
 
 	struct lio_sriov_info sriov_info;
 
+	/** Mail Box details of each lio queue. */
+	struct lio_mbox **mbox;
+
 	char dev_string[LIO_DEVICE_NAME_LEN]; /* Device print string */
 
 	const struct lio_config *default_config;
-- 
1.8.3.1