[dpdk-dev] [PATCH v3 07/27] common/octeontx2: introduce common device class

jerinj at marvell.com jerinj at marvell.com
Mon Jun 17 17:55:17 CEST 2019


From: Jerin Jacob <jerinj at marvell.com>

Introduce otx2_dev class to hold octeontx2 PCIe device specific
information and operations.

All PCIe drivers(ethdev, mempool, cryptodev and eventdev) in octeontx2,
inherits this base object to avail the common functionalities such
as mailbox creation, interrupt registration, etc of the PCIe device.

Signed-off-by: Jerin Jacob <jerinj at marvell.com>
Signed-off-by: Nithin Dabilpuram <ndabilpuram at marvell.com>
---
 drivers/common/octeontx2/Makefile             |   2 +
 drivers/common/octeontx2/meson.build          |   4 +-
 drivers/common/octeontx2/otx2_common.h        |  14 ++
 drivers/common/octeontx2/otx2_dev.c           | 197 ++++++++++++++++++
 drivers/common/octeontx2/otx2_dev.h           |  84 ++++++++
 drivers/common/octeontx2/otx2_irq.h           |  19 ++
 .../rte_common_octeontx2_version.map          |   3 +
 7 files changed, 321 insertions(+), 2 deletions(-)
 create mode 100644 drivers/common/octeontx2/otx2_dev.c
 create mode 100644 drivers/common/octeontx2/otx2_dev.h
 create mode 100644 drivers/common/octeontx2/otx2_irq.h

diff --git a/drivers/common/octeontx2/Makefile b/drivers/common/octeontx2/Makefile
index 3fd67f0ab..a6f94553d 100644
--- a/drivers/common/octeontx2/Makefile
+++ b/drivers/common/octeontx2/Makefile
@@ -11,6 +11,7 @@ LIB = librte_common_octeontx2.a
 
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/common/octeontx2
+CFLAGS += -I$(RTE_SDK)/drivers/bus/pci
 
 ifneq ($(CONFIG_RTE_ARCH_64),y)
 CFLAGS += -Wno-int-to-pointer-cast
@@ -24,6 +25,7 @@ LIBABIVER := 1
 #
 # all source are stored in SRCS-y
 #
+SRCS-y += otx2_dev.c
 SRCS-y += otx2_mbox.c
 SRCS-y += otx2_common.c
 
diff --git a/drivers/common/octeontx2/meson.build b/drivers/common/octeontx2/meson.build
index 4771b1942..feaf75d92 100644
--- a/drivers/common/octeontx2/meson.build
+++ b/drivers/common/octeontx2/meson.build
@@ -2,7 +2,7 @@
 # Copyright(C) 2019 Marvell International Ltd.
 #
 
-sources= files(
+sources= files('otx2_dev.c',
 		'otx2_mbox.c',
 		'otx2_common.c',
 	       )
@@ -19,6 +19,6 @@ foreach flag: extra_flags
 	endif
 endforeach
 
-deps = ['eal', 'ethdev']
+deps = ['eal', 'pci', 'ethdev']
 includes += include_directories('../../common/octeontx2',
 		'../../bus/pci')
diff --git a/drivers/common/octeontx2/otx2_common.h b/drivers/common/octeontx2/otx2_common.h
index 58fcf5a41..b9e7a7f8d 100644
--- a/drivers/common/octeontx2/otx2_common.h
+++ b/drivers/common/octeontx2/otx2_common.h
@@ -69,6 +69,20 @@ extern int otx2_logtype_dpi;
 #define otx2_tim_dbg(fmt, ...) otx2_dbg(tim, fmt, ##__VA_ARGS__)
 #define otx2_dpi_dbg(fmt, ...) otx2_dbg(dpi, fmt, ##__VA_ARGS__)
 
+/* PCI IDs */
+#define PCI_VENDOR_ID_CAVIUM			0x177D
+#define PCI_DEVID_OCTEONTX2_RVU_PF              0xA063
+#define PCI_DEVID_OCTEONTX2_RVU_VF		0xA064
+#define PCI_DEVID_OCTEONTX2_RVU_AF		0xA065
+#define PCI_DEVID_OCTEONTX2_RVU_SSO_TIM_PF	0xA0F9
+#define PCI_DEVID_OCTEONTX2_RVU_SSO_TIM_VF	0xA0FA
+#define PCI_DEVID_OCTEONTX2_RVU_NPA_PF		0xA0FB
+#define PCI_DEVID_OCTEONTX2_RVU_NPA_VF		0xA0FC
+#define PCI_DEVID_OCTEONTX2_RVU_CPT_PF		0xA0FD
+#define PCI_DEVID_OCTEONTX2_RVU_CPT_VF		0xA0FE
+#define PCI_DEVID_OCTEONTX2_RVU_AF_VF		0xA0f8
+#define PCI_DEVID_OCTEONTX2_DPI_VF		0xA081
+
 /* IO Access */
 #define otx2_read64(addr) rte_read64_relaxed((void *)(addr))
 #define otx2_write64(val, addr) rte_write64_relaxed((val), (void *)(addr))
diff --git a/drivers/common/octeontx2/otx2_dev.c b/drivers/common/octeontx2/otx2_dev.c
new file mode 100644
index 000000000..486b1b7c8
--- /dev/null
+++ b/drivers/common/octeontx2/otx2_dev.c
@@ -0,0 +1,197 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2019 Marvell International Ltd.
+ */
+
+#include <fcntl.h>
+#include <inttypes.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_eal.h>
+#include <rte_memcpy.h>
+
+#include "otx2_dev.h"
+#include "otx2_mbox.h"
+
+/* PF/VF message handling timer */
+#define VF_PF_MBOX_TIMER_MS	(20 * 1000)
+
+static void *
+mbox_mem_map(off_t off, size_t size)
+{
+	void *va = MAP_FAILED;
+	int mem_fd;
+
+	if (size <= 0)
+		goto error;
+
+	mem_fd = open("/dev/mem", O_RDWR);
+	if (mem_fd < 0)
+		goto error;
+
+	va = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, mem_fd, off);
+	close(mem_fd);
+
+	if (va == MAP_FAILED)
+		otx2_err("Failed to mmap sz=0x%zx, fd=%d, off=%jd",
+			 size, mem_fd, (intmax_t)off);
+error:
+	return va;
+}
+
+static void
+mbox_mem_unmap(void *va, size_t size)
+{
+	if (va)
+		munmap(va, size);
+}
+
+static void
+otx2_update_pass_hwcap(struct rte_pci_device *pci_dev, struct otx2_dev *dev)
+{
+	RTE_SET_USED(pci_dev);
+
+	/* Update this logic when we have A1 */
+	dev->hwcap |= OTX2_HWCAP_F_A0;
+}
+
+static void
+otx2_update_vf_hwcap(struct rte_pci_device *pci_dev, struct otx2_dev *dev)
+{
+	dev->hwcap = 0;
+
+	switch (pci_dev->id.device_id) {
+	case PCI_DEVID_OCTEONTX2_RVU_PF:
+		break;
+	case PCI_DEVID_OCTEONTX2_RVU_SSO_TIM_VF:
+	case PCI_DEVID_OCTEONTX2_RVU_NPA_VF:
+	case PCI_DEVID_OCTEONTX2_RVU_CPT_VF:
+	case PCI_DEVID_OCTEONTX2_RVU_AF_VF:
+	case PCI_DEVID_OCTEONTX2_RVU_VF:
+		dev->hwcap |= OTX2_HWCAP_F_VF;
+		break;
+	}
+}
+
+/**
+ * @internal
+ * Initialize the otx2 device
+ */
+int
+otx2_dev_init(struct rte_pci_device *pci_dev, void *otx2_dev)
+{
+	int up_direction = MBOX_DIR_PFAF_UP;
+	int rc, direction = MBOX_DIR_PFAF;
+	struct otx2_dev *dev = otx2_dev;
+	uintptr_t bar2, bar4;
+	uint64_t bar4_addr;
+	void *hwbase;
+
+	bar2 = (uintptr_t)pci_dev->mem_resource[2].addr;
+	bar4 = (uintptr_t)pci_dev->mem_resource[4].addr;
+
+	if (bar2 == 0 || bar4 == 0) {
+		otx2_err("Failed to get pci bars");
+		rc = -ENODEV;
+		goto error;
+	}
+
+	dev->node = pci_dev->device.numa_node;
+	dev->maxvf = pci_dev->max_vfs;
+	dev->bar2 = bar2;
+	dev->bar4 = bar4;
+
+	otx2_update_vf_hwcap(pci_dev, dev);
+	otx2_update_pass_hwcap(pci_dev, dev);
+
+	if (otx2_dev_is_vf(dev)) {
+		direction = MBOX_DIR_VFPF;
+		up_direction = MBOX_DIR_VFPF_UP;
+	}
+
+	/* Initialize the local mbox */
+	rc = otx2_mbox_init(&dev->mbox_local, bar4, bar2, direction, 1);
+	if (rc)
+		goto error;
+	dev->mbox = &dev->mbox_local;
+
+	rc = otx2_mbox_init(&dev->mbox_up, bar4, bar2, up_direction, 1);
+	if (rc)
+		goto error;
+
+	/* Check the readiness of PF/VF */
+	rc = otx2_send_ready_msg(dev->mbox, &dev->pf_func);
+	if (rc)
+		goto mbox_fini;
+
+	dev->pf = otx2_get_pf(dev->pf_func);
+	dev->vf = otx2_get_vf(dev->pf_func);
+	memset(&dev->active_vfs, 0, sizeof(dev->active_vfs));
+
+	/* Found VF devices in a PF device */
+	if (pci_dev->max_vfs > 0) {
+
+		/* Remap mbox area for all vf's */
+		bar4_addr = otx2_read64(bar2 + RVU_PF_VF_BAR4_ADDR);
+		if (bar4_addr == 0) {
+			rc = -ENODEV;
+			goto mbox_fini;
+		}
+
+		hwbase = mbox_mem_map(bar4_addr, MBOX_SIZE * pci_dev->max_vfs);
+		if (hwbase == MAP_FAILED) {
+			rc = -ENOMEM;
+			goto mbox_fini;
+		}
+		/* Init mbox object */
+		rc = otx2_mbox_init(&dev->mbox_vfpf, (uintptr_t)hwbase,
+				    bar2, MBOX_DIR_PFVF, pci_dev->max_vfs);
+		if (rc)
+			goto iounmap;
+
+		/* PF -> VF UP messages */
+		rc = otx2_mbox_init(&dev->mbox_vfpf_up, (uintptr_t)hwbase,
+				    bar2, MBOX_DIR_PFVF_UP, pci_dev->max_vfs);
+		if (rc)
+			goto mbox_fini;
+	}
+
+	dev->mbox_active = 1;
+	return rc;
+
+iounmap:
+	mbox_mem_unmap(hwbase, MBOX_SIZE * pci_dev->max_vfs);
+mbox_fini:
+	otx2_mbox_fini(dev->mbox);
+	otx2_mbox_fini(&dev->mbox_up);
+error:
+	return rc;
+}
+
+/**
+ * @internal
+ * Finalize the otx2 device
+ */
+void
+otx2_dev_fini(struct rte_pci_device *pci_dev, void *otx2_dev)
+{
+	struct otx2_dev *dev = otx2_dev;
+	struct otx2_mbox *mbox;
+
+	/* Release PF - VF */
+	mbox = &dev->mbox_vfpf;
+	if (mbox->hwbase && mbox->dev)
+		mbox_mem_unmap((void *)mbox->hwbase,
+			       MBOX_SIZE * pci_dev->max_vfs);
+	otx2_mbox_fini(mbox);
+	mbox = &dev->mbox_vfpf_up;
+	otx2_mbox_fini(mbox);
+
+	/* Release PF - AF */
+	mbox = dev->mbox;
+	otx2_mbox_fini(mbox);
+	mbox = &dev->mbox_up;
+	otx2_mbox_fini(mbox);
+	dev->mbox_active = 0;
+}
diff --git a/drivers/common/octeontx2/otx2_dev.h b/drivers/common/octeontx2/otx2_dev.h
new file mode 100644
index 000000000..a89570b62
--- /dev/null
+++ b/drivers/common/octeontx2/otx2_dev.h
@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2019 Marvell International Ltd.
+ */
+
+#ifndef _OTX2_DEV_H
+#define _OTX2_DEV_H
+
+#include <rte_bus_pci.h>
+
+#include "otx2_common.h"
+#include "otx2_irq.h"
+#include "otx2_mbox.h"
+
+/* Common HWCAP flags. Use from LSB bits */
+#define OTX2_HWCAP_F_VF		BIT_ULL(0) /* VF device */
+#define otx2_dev_is_vf(dev)	(dev->hwcap & OTX2_HWCAP_F_VF)
+#define otx2_dev_is_pf(dev)	(!(dev->hwcap & OTX2_HWCAP_F_VF))
+#define otx2_dev_is_lbk(dev)	((dev->hwcap & OTX2_HWCAP_F_VF) && \
+				 (dev->tx_chan_base < 0x700))
+
+#define OTX2_HWCAP_F_A0		BIT_ULL(1) /* A0 device */
+#define otx2_dev_is_A0(dev)	(dev->hwcap & OTX2_HWCAP_F_A0)
+
+struct otx2_dev;
+
+#define OTX2_DEV					\
+	int node __rte_cache_aligned;			\
+	uint16_t pf;					\
+	int16_t vf;					\
+	uint16_t pf_func;				\
+	uint8_t mbox_active;				\
+	bool drv_inited;				\
+	uint64_t active_vfs[MAX_VFPF_DWORD_BITS];	\
+	uintptr_t bar2;					\
+	uintptr_t bar4;					\
+	struct otx2_mbox mbox_local;			\
+	struct otx2_mbox mbox_up;			\
+	struct otx2_mbox mbox_vfpf;			\
+	struct otx2_mbox mbox_vfpf_up;			\
+	otx2_intr_t intr;				\
+	int timer_set;	/* ~0 : no alarm handling */	\
+	uint64_t hwcap;					\
+	struct otx2_mbox *mbox;				\
+	uint16_t maxvf;					\
+	const struct otx2_dev_ops *ops
+
+struct otx2_dev {
+	OTX2_DEV;
+};
+
+int otx2_dev_init(struct rte_pci_device *pci_dev, void *otx2_dev);
+void otx2_dev_fini(struct rte_pci_device *pci_dev, void *otx2_dev);
+int otx2_dev_active_vfs(void *otx2_dev);
+
+#define RVU_PFVF_PF_SHIFT	10
+#define RVU_PFVF_PF_MASK	0x3F
+#define RVU_PFVF_FUNC_SHIFT	0
+#define RVU_PFVF_FUNC_MASK	0x3FF
+
+static inline int
+otx2_get_vf(uint16_t pf_func)
+{
+	return (((pf_func >> RVU_PFVF_FUNC_SHIFT) & RVU_PFVF_FUNC_MASK) - 1);
+}
+
+static inline int
+otx2_get_pf(uint16_t pf_func)
+{
+	return (pf_func >> RVU_PFVF_PF_SHIFT) & RVU_PFVF_PF_MASK;
+}
+
+static inline int
+otx2_pfvf_func(int pf, int vf)
+{
+	return (pf << RVU_PFVF_PF_SHIFT) | ((vf << RVU_PFVF_FUNC_SHIFT) + 1);
+}
+
+static inline int
+otx2_is_afvf(uint16_t pf_func)
+{
+	return !(pf_func & ~RVU_PFVF_FUNC_MASK);
+}
+
+#endif /* _OTX2_DEV_H */
diff --git a/drivers/common/octeontx2/otx2_irq.h b/drivers/common/octeontx2/otx2_irq.h
new file mode 100644
index 000000000..df44ddfba
--- /dev/null
+++ b/drivers/common/octeontx2/otx2_irq.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2019 Marvell International Ltd.
+ */
+
+#ifndef _OTX2_IRQ_H_
+#define _OTX2_IRQ_H_
+
+#include <rte_pci.h>
+#include <rte_interrupts.h>
+
+#include "otx2_common.h"
+
+typedef struct {
+/* 128 devices translate to two 64 bits dwords */
+#define MAX_VFPF_DWORD_BITS 2
+	uint64_t bits[MAX_VFPF_DWORD_BITS];
+} otx2_intr_t;
+
+#endif /* _OTX2_IRQ_H_ */
diff --git a/drivers/common/octeontx2/rte_common_octeontx2_version.map b/drivers/common/octeontx2/rte_common_octeontx2_version.map
index e10a2d3b2..4d9879899 100644
--- a/drivers/common/octeontx2/rte_common_octeontx2_version.map
+++ b/drivers/common/octeontx2/rte_common_octeontx2_version.map
@@ -1,6 +1,9 @@
 DPDK_19.08 {
 	global:
 
+	otx2_dev_fini;
+	otx2_dev_init;
+
 	otx2_logtype_base;
 	otx2_logtype_dpi;
 	otx2_logtype_mbox;
-- 
2.21.0



More information about the dev mailing list