[dpdk-dev] [RFC][PATCH V2 1/3] examples/vhost: Add vswitch (generic switch) framework
Pankaj Chauhan
pankaj.chauhan at nxp.com
Mon Sep 5 12:54:29 CEST 2016
Introduce support for a generic framework for handling of switching between
physical and vhost devices. The vswitch framework introduces the following
concept:
1. vswitch_dev: Vswitch device is a logical switch which can have physical and
virtio devices. The devices are operated/used using standard rte_eth API for
physical devices and lib_vhost API for vhost devices, PMD API is not used
for vhost devices.
2. vswitch_port: Any physical or virtio device that is added to vswitch. The
port can have its own tx/rx functions for doing data transfer, which are exposed
to the framework using generic function pointers (vs_port->do_tx/do_rx). This way
the generic code can do tx/rx without understanding the type of device (Physical or
virtio). Similarly each port has its own functions to select tx/rx queues. The framework
provides default tx/rx queue selection functions to the port when port is added
(for both physical and vhost devices). But the framework allows the switch implementation
to override the queue selection functions (vs_port->get_txq/rxq) if required.
3. vswitch_ops: The ops is set of function pointers which are used to do operations
like learning, unlearning, add/delete port, lookup_and_forward. The user of vswitch
framework (vhost/main.[c,h])uses these function pointers to perform above mentioned
operations, thus it remains agnostic of the underlying implementation.
Different switching logics can implement their vswitch_device and vswitch_ops, and
register with the framework. This framework makes vhost-switch application scalable
in terms of:
1. Different switching logics (one of them is vmdq, vhost/vmdq.[c,h]
2. Number of ports.
3. Policies of selecting ports for rx and tx.
Signed-off-by: Pankaj Chauhan <pankaj.chauhan at nxp.com>
---
examples/vhost/Makefile | 2 +-
examples/vhost/main.c | 128 +++++++++--
examples/vhost/vswitch_common.c | 499 ++++++++++++++++++++++++++++++++++++++++
examples/vhost/vswitch_common.h | 186 +++++++++++++++
examples/vhost/vswitch_txrx.c | 97 ++++++++
examples/vhost/vswitch_txrx.h | 71 ++++++
6 files changed, 966 insertions(+), 17 deletions(-)
create mode 100644 examples/vhost/vswitch_common.c
create mode 100644 examples/vhost/vswitch_common.h
create mode 100644 examples/vhost/vswitch_txrx.c
create mode 100644 examples/vhost/vswitch_txrx.h
diff --git a/examples/vhost/Makefile b/examples/vhost/Makefile
index e95c68a..1b58308 100644
--- a/examples/vhost/Makefile
+++ b/examples/vhost/Makefile
@@ -48,7 +48,7 @@ else
APP = vhost-switch
# all source are stored in SRCS-y
-SRCS-y := main.c
+SRCS-y := main.c vswitch_common.c vswitch_txrx.c vmdq.c
CFLAGS += -O2 -D_FILE_OFFSET_BITS=64
CFLAGS += $(WERROR_FLAGS)
diff --git a/examples/vhost/main.c b/examples/vhost/main.c
index 92a9823..c949df4 100644
--- a/examples/vhost/main.c
+++ b/examples/vhost/main.c
@@ -811,9 +811,16 @@ static inline void __attribute__((always_inline))
virtio_xmit(struct vhost_dev *dst_vdev, struct vhost_dev *src_vdev,
struct rte_mbuf *m)
{
- uint16_t ret;
+ uint16_t ret, txq;
+ struct vswitch_port *vs_port = dst_vdev->vs_port;
- ret = rte_vhost_enqueue_burst(dst_vdev->vid, VIRTIO_RXQ, &m, 1);
+ /* The switch implmentation decides the txq for each port itself.
+ * this gives switch implmentations to choose thier own queue management
+ * for each port
+ */
+
+ txq = vs_port->get_txq(vs_port, rte_lcore_id());
+ ret = rte_vhost_enqueue_burst(dst_vdev->vid, txq, &m, 1);
if (enable_stats) {
rte_atomic64_inc(&dst_vdev->stats.rx_total_atomic);
rte_atomic64_add(&dst_vdev->stats.rx_atomic, ret);
@@ -832,7 +839,7 @@ virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)
struct ether_hdr *pkt_hdr;
struct vhost_dev *dst_vdev;
- pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
+ pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
dst_vdev = find_vhost_dev(&pkt_hdr->d_addr);
if (!dst_vdev)
@@ -934,11 +941,26 @@ static inline void __attribute__((always_inline))
do_drain_mbuf_table(struct mbuf_table *tx_q)
{
uint16_t count;
+ struct vswitch_port *tx_port;
+
+ /* Let switch implmentation decide which physical port to do tx to.
+ * Every switch implmentation may have it's own strategy, for example
+ * VMDQ does tx to only one Physical port. Having a scheduler function
+ * which is switch specefic give flexibility to have another strategy
+ * for a switch
+ */
+ tx_port = vs_sched_tx_port(vswitch_dev_g, VSWITCH_PTYPE_PHYS,
+ rte_lcore_id());
+ if (unlikely(!tx_port))
+ goto out;
- count = rte_eth_tx_burst(ports[0], tx_q->txq_id,
- tx_q->m_table, tx_q->len);
- if (unlikely(count < tx_q->len))
- free_pkts(&tx_q->m_table[count], tx_q->len - count);
+ if (tx_q->len) {
+ count = tx_port->do_tx(tx_port, tx_q->txq_id, NULL, tx_q->m_table,
+ tx_q->len);
+
+ if (unlikely(count < tx_q->len))
+ free_pkts(&tx_q->m_table[count], tx_q->len - count);
+ }
tx_q->len = 0;
}
@@ -1060,6 +1082,28 @@ drain_eth_rx(struct vhost_dev *vdev)
{
uint16_t rx_count, enqueue_count;
struct rte_mbuf *pkts[MAX_PKT_BURST];
+ uint16_t rxq, core_id;
+ struct vswitch_port *rx_port;
+
+
+ core_id = rte_lcore_id();
+ /* Let switch implmentation decide which physical port to do rx from.
+ * Every switch implmentation may have it's own strategy, for example
+ * VMDQ does rx from only one Physical port. Having a scheduler function
+ * which is switch specefic give flexibility to have another strategy
+ * for a switch
+ */
+ rx_port = vs_sched_rx_port(vswitch_dev_g, VSWITCH_PTYPE_PHYS,
+ core_id);
+ if (unlikely(!rx_port))
+ goto out;
+
+ /* The switch implmentation decides the rxq for each port itself.
+ * this gives switch implmentations to choose thier own queue management
+ * for each port
+ */
+ rxq = rx_port->get_rxq(rx_port, vdev, core_id);
+ rx_count = rx_port->do_rx(rx_port, rxq, NULL, pkts, MAX_PKT_BURST);
rx_count = rte_eth_rx_burst(ports[0], vdev->vmdq_rx_q,
pkts, MAX_PKT_BURST);
@@ -1098,20 +1142,31 @@ static inline void __attribute__((always_inline))
drain_virtio_tx(struct vhost_dev *vdev)
{
struct rte_mbuf *pkts[MAX_PKT_BURST];
- uint16_t count;
- uint16_t i;
+ struct vswitch_port *vs_port = vdev->vs_port;
+ uint16_t count, rxq, core_id;
+
+ /* The switch implmentation decides the rxq for each port itself.
+ * this gives switch implmentations to choose thier own queue management
+ * for each port
+ */
- count = rte_vhost_dequeue_burst(vdev->vid, VIRTIO_TXQ, mbuf_pool,
+ core_id = rte_lcore_id();
+ rxq = vs_port->get_rxq(vs_port, NULL, core_id);
+ count = rte_vhost_dequeue_burst(vdev->vid, rxq, mbuf_pool,
pkts, MAX_PKT_BURST);
- /* setup VMDq for the first packet */
- if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && count) {
- if (vdev->remove || link_vmdq(vdev, pkts[0]) == -1)
+ if (unlikely(!count))
+ goto out;
+
+ if (unlikely(vdev->ready == DEVICE_MAC_LEARNING)) {
+ if (vdev->remove || vs_learn_port(vs_port, pkts, count))
free_pkts(pkts, count);
}
- for (i = 0; i < count; ++i)
- virtio_tx_route(vdev, pkts[i], vlan_tags[vdev->vid]);
+ vs_lookup_n_fwd(vs_port, pkts, count, VIRTIO_RXQ);
+
+out:
+ return;
}
/*
@@ -1241,7 +1296,7 @@ static int
new_device(int vid)
{
int lcore, core_add = 0;
- uint32_t device_num_min = num_devices;
+ uint32_t device_num_min;
struct vhost_dev *vdev;
vdev = rte_zmalloc("vhost device", sizeof(*vdev), RTE_CACHE_LINE_SIZE);
@@ -1252,6 +1307,16 @@ new_device(int vid)
return -1;
}
vdev->vid = vid;
+ device_num_min = vs_get_max_vdevs(vswitch_dev_g);
+ RTE_LOG(INFO, VHOST_PORT, "max virtio devices %d\n", device_num_min);
+
+ vs_port = vs_add_port(vswitch_dev_g, vid, VSWITCH_PTYPE_VIRTIO, vdev);
+
+ if (!vs_port) {
+ rte_exit(EXIT_FAILURE, "Failed to add port [%d] to vsdev %s\n",
+ vs_port->port_id, vswitch_dev_g->name);
+ }
+ vdev->vs_port = vs_port;
TAILQ_INSERT_TAIL(&vhost_dev_list, vdev, global_vdev_entry);
vdev->vmdq_rx_q = vid * queues_per_pool + vmdq_queue_base;
@@ -1277,6 +1342,10 @@ new_device(int vid)
rte_vhost_enable_guest_notification(vid, VIRTIO_RXQ, 0);
rte_vhost_enable_guest_notification(vid, VIRTIO_TXQ, 0);
+ RTE_LOG(INFO, VHOST_PORT, "New virtio port %d\n", vid);
+
+ vs_port_start(vs_port);
+
RTE_LOG(INFO, VHOST_DATA,
"(%d) device has been added to data core %d\n",
vid, vdev->coreid);
@@ -1402,6 +1471,31 @@ create_mbuf_pool(uint16_t nr_port, uint32_t nr_switch_core, uint32_t mbuf_size,
rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
}
+static uint64_t get_vswitch_conf_flags(void)
+{
+ uint64_t vs_conf_flags = 0;
+
+ if (vm2vm_mode == VM2VM_HARDWARE)
+ vs_conf_flags |= VS_CNF_FLG_VM2VM_HARDWARE;
+
+ if (vm2vm_mode == VM2VM_SOFTWARE)
+ vs_conf_flags |= VS_CNF_FLG_VM2VM_SOFTWARE;
+
+ if (promiscuous)
+ vs_conf_flags |= VS_CNF_FLG_PROMISCOUS_EN;
+
+ if (jumbo_frame_en)
+ vs_conf_flags |= VS_CNF_FLG_JUMBO_EN;
+
+ if (enable_stats)
+ vs_conf_flags |= VS_CNF_FLG_STATS_EN;
+
+ if (vlan_strip)
+ vs_conf_flags |= VS_CNF_FLG_VLAN_STRIP_EN;
+
+ return vs_conf_flags;
+}
+
/*
* Main function, does initialisation and calls the per-lcore functions. The CUSE
* device is also registered here to handle the IOCTLs.
@@ -1484,6 +1578,7 @@ main(int argc, char *argv[])
"Cannot initialize network ports\n");
}
+ RTE_LOG(INFO, VHOST_CONFIG, "PORT init done \n");
/* Enable stats if the user option is set. */
if (enable_stats) {
ret = pthread_create(&tid, NULL, (void *)print_stats, NULL);
@@ -1499,6 +1594,7 @@ main(int argc, char *argv[])
"Cannot set print-stats name\n");
}
+ RTE_LOG(INFO, VHOST_CONFIG, "Launching switch_worker threads \n");
/* Launch all data cores. */
RTE_LCORE_FOREACH_SLAVE(lcore_id)
rte_eal_remote_launch(switch_worker, NULL, lcore_id);
diff --git a/examples/vhost/vswitch_common.c b/examples/vhost/vswitch_common.c
new file mode 100644
index 0000000..9f9ab87
--- /dev/null
+++ b/examples/vhost/vswitch_common.c
@@ -0,0 +1,499 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2016 Freescale Semiconductor. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Freescale Semiconductor nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <unistd.h>
+#include <rte_atomic.h>
+#include <rte_cycles.h>
+#include <rte_ethdev.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_virtio_net.h>
+#include <rte_ip.h>
+#include <rte_tcp.h>
+
+#include <sys/queue.h>
+#include <strings.h>
+#include <errno.h>
+
+#include "vswitch_common.h"
+#include "vswitch_txrx.h"
+
+/* Meta data for vswitch. Since this is going to be used in this file only it
+ * is defined here instead of in a header file.
+ */
+struct vs_mdata {
+ rte_spinlock_t lock;
+ int dev_count;
+ LIST_HEAD(, vswitch_dev) head;
+};
+
+static struct vs_mdata *vs_mdata_g;
+
+struct vswitch_dev *vs_get_vswitch_dev(const char *name)
+{
+ struct vswitch_dev *temp, *vs_dev = NULL;
+
+ rte_spinlock_lock(&vs_mdata_g->lock);
+ LIST_FOREACH(temp, &vs_mdata_g->head, list) {
+ if (!strncmp(temp->name, name, VSWITCH_NAME_SIZE))
+ vs_dev = temp;
+ }
+ rte_spinlock_unlock(&vs_mdata_g->lock);
+
+ return vs_dev;
+}
+
+static struct vswitch_port *vs_get_free_port(struct vswitch_dev *vs_dev)
+{
+ int i, found = 0;
+ struct vswitch_port *vs_port = NULL;
+
+ rte_spinlock_lock(&vs_dev->lock);
+
+ for (i = 0; i < vs_dev->port_count; i++) {
+ vs_port = &vs_dev->ports[i];
+ if (vs_port->state == VSWITCH_PSTATE_NOT_INUSE) {
+ found = 1;
+ break;
+ }
+ }
+
+ if (found)
+ vs_port->state = VSWITCH_PSTATE_BEING_ALLOCATED;
+ else
+ vs_port = NULL;
+
+ rte_spinlock_unlock(&vs_dev->lock);
+ return vs_port;
+}
+
+static void vs_free_port(struct vswitch_port *vs_port)
+{
+ struct vswitch_dev *vs_dev = vs_port->vs_dev;
+
+ rte_spinlock_lock(&vs_dev->lock);
+ vs_port->state = VSWITCH_PSTATE_NOT_INUSE;
+ rte_spinlock_unlock(&vs_dev->lock);
+}
+
+static __attribute__((unused))struct vswitch_port *vs_get_port(
+ struct vswitch_dev *vs_dev, unsigned int port_id,
+ enum vswitch_port_type type)
+{
+ int i;
+ struct vswitch_port *vs_port = NULL;
+
+ for (i = 0; i < vs_dev->port_count; i++) {
+ vs_port = &vs_dev->ports[i];
+ if ((vs_port->port_id == port_id) && (vs_port->type == type))
+ return vs_port;
+ }
+
+ return NULL;
+}
+
+int vs_port_start(struct vswitch_port *vs_port)
+{
+ struct vswitch_ops *vs_ops = vs_port->vs_dev->ops;
+ int rc = 0;
+
+ if (vs_ops->port_start)
+ rc = vs_ops->port_start(vs_port);
+
+ if (!rc)
+ vs_port->state = VSWITCH_PSTATE_LEARNING;
+
+ return rc;
+}
+
+int vs_port_stop(struct vswitch_port *vs_port)
+{
+ struct vswitch_ops *vs_ops = vs_port->vs_dev->ops;
+ int rc = 0;
+
+ if (vs_ops->port_stop)
+ rc = vs_ops->port_stop(vs_port);
+
+ if (!rc)
+ vs_port->state = VSWITCH_PSTATE_ADDED;
+
+ return rc;
+}
+
+struct vswitch_port *vs_add_port(struct vswitch_dev *vs_dev, int port_id,
+ enum vswitch_port_type type, void *priv)
+{
+ int rc = 0;
+ struct vswitch_port *vs_port = NULL;
+ struct vswitch_ops *vs_ops = vs_dev->ops;
+
+ vs_port = vs_get_free_port(vs_dev);
+ if (!vs_port) {
+ RTE_LOG(DEBUG, VHOST_CONFIG, "Failed get free port in \
+ vswitch %s\n", vs_dev->name);
+ rc = -EBUSY;
+ goto out;
+ }
+
+ vs_port->port_id = port_id;
+ vs_port->type = type;
+ vs_port->priv = priv;
+
+ /* Initialize default port operations. It should be noted that
+ * The switch ops->add_port can replace them with switch specefic
+ * operations if required. This gives us more flexibility in switch
+ * implementations.
+ */
+
+ switch (type) {
+ case VSWITCH_PTYPE_PHYS:
+ vs_port->do_tx = vs_do_tx_phys_port;
+ vs_port->do_rx = vs_do_rx_phys_port;
+ vs_port->get_txq = vs_get_txq_phys_port;
+ vs_port->get_rxq = vs_get_rxq_phys_port;
+ break;
+ case VSWITCH_PTYPE_VIRTIO:
+ vs_port->do_tx = vs_do_tx_virtio_port;
+ vs_port->do_rx = vs_do_rx_virtio_port;
+ vs_port->get_txq = vs_get_txq_virtio_port;
+ vs_port->get_rxq = vs_get_rxq_virtio_port;
+ break;
+ default:
+ RTE_LOG(DEBUG, VHOST_CONFIG, "Invalid port [id %d, type %d]",
+ port_id, type);
+ rc = -EINVAL;
+ goto out;
+ }
+
+ if (vs_ops->add_port)
+ rc = vs_ops->add_port(vs_port);
+
+ if (rc)
+ goto out;
+
+ vs_port->state = VSWITCH_PSTATE_ADDED;
+
+ rte_eth_macaddr_get(vs_port->port_id, &vs_port->mac_addr);
+ RTE_LOG(INFO, VHOST_PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
+ " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
+ (unsigned)port_id,
+ vs_port->mac_addr.addr_bytes[0],
+ vs_port->mac_addr.addr_bytes[1],
+ vs_port->mac_addr.addr_bytes[2],
+ vs_port->mac_addr.addr_bytes[3],
+ vs_port->mac_addr.addr_bytes[4],
+ vs_port->mac_addr.addr_bytes[5]);
+
+ RTE_LOG(DEBUG, VHOST_CONFIG, "Added port [%d, type %d] to \
+ vswitch %s\n", vs_port->port_id, type, vs_dev->name);
+out:
+ if (rc){
+ RTE_LOG(INFO, VHOST_CONFIG, "Failed to Add port [%d, type %d] to \
+ vswitch %s\n", port_id, type, vs_dev->name);
+ if (vs_port)
+ vs_free_port(vs_port);
+ vs_port = NULL;
+ }
+
+ return vs_port;
+}
+
+int vs_del_port(struct vswitch_port *vs_port)
+{
+ int rc = 0;
+ struct vswitch_ops *vs_ops = vs_port->vs_dev->ops;
+
+ if (vs_ops->del_port)
+ rc = vs_ops->del_port(vs_port);
+
+ if (!rc)
+ vs_port->state = VSWITCH_PSTATE_NOT_INUSE;
+
+
+ /*TBD:XXX: may be put a dummy function which frees all packets without
+ * any tx/rx, NULL looks ugly!
+ */
+ vs_port->do_tx = vs_port->do_rx = NULL;
+
+ RTE_LOG(DEBUG, VHOST_CONFIG, "Removed port [%d, type %d] from \
+ vswitch %s\n", vs_port->port_id, vs_port->type,
+ vs_port->vs_dev->name);
+
+ return rc;
+}
+
+int vs_learn_port(struct vswitch_port *vs_port, struct rte_mbuf
+ **pkts, uint16_t count)
+{
+ struct vswitch_ops *vs_ops = vs_port->vs_dev->ops;
+ int rc;
+
+ rc = vs_ops->learn_port(vs_port, pkts, count);
+ if (!rc)
+ vs_port->state = VSWITCH_PSTATE_FORWARDING;
+
+ return rc;
+}
+
+int vs_unlearn_port(struct vswitch_port *vs_port)
+{
+ struct vswitch_ops *vs_ops = vs_port->vs_dev->ops;
+
+ vs_ops->unlearn_port(vs_port);
+ vs_port->state = VSWITCH_PSTATE_LEARNING;
+
+ return 0;
+}
+
+
+int vs_lookup_n_fwd(struct vswitch_port *vs_port, struct rte_mbuf **pkts,
+ uint16_t count, uint16_t in_rxq)
+{
+ int rc, i;
+ struct vswitch_ops *vs_ops = vs_port->vs_dev->ops;
+
+ rc = vs_ops->lookup_n_fwd(vs_port, pkts, count, in_rxq);
+
+ if (rc) {
+ for (i = 0; i < count; i++)
+ rte_pktmbuf_free(pkts[i]);
+ }
+
+ return rc;
+}
+
+int vs_do_broadcast_fwd(struct vswitch_dev *vs_dev,
+ struct vswitch_port *in_port, uint32_t txport_type_mask,
+ struct rte_mbuf *mbuf)
+{
+ int i = 0, tx_q;
+ struct vswitch_port *dest_port;
+
+ for (i = 0; i < vs_dev->port_count; i++) {
+ dest_port = &vs_dev->ports[i];
+ /*Do not broadcast on incomming port */
+ if ((in_port->type == dest_port->type) &&
+ (in_port->port_id == dest_port->port_id))
+ continue;
+ if (!(VS_PTYPE_MASK(dest_port->type) & txport_type_mask))
+ continue;
+
+ tx_q = dest_port->get_txq(dest_port, rte_lcore_id());
+ dest_port->do_tx(dest_port, tx_q, NULL, &mbuf, 1);
+ }
+
+ return 0;
+}
+
+struct vswitch_port *vs_sched_rx_port(struct vswitch_dev *vs_dev, enum
+ vswitch_port_type ptype, uint16_t core_id)
+{
+ struct vswitch_port *vs_port = NULL;
+ struct vswitch_ops *vs_ops = vs_dev->ops;
+
+ if (vs_ops->sched_rx_port)
+ vs_port = vs_ops->sched_rx_port(vs_dev, ptype, core_id);
+
+ return vs_port;
+}
+
+struct vswitch_port *vs_sched_tx_port(struct vswitch_dev *vs_dev, enum
+ vswitch_port_type ptype, uint16_t core_id)
+{
+ struct vswitch_port *vs_port = NULL;
+ struct vswitch_ops *vs_ops = vs_dev->ops;
+
+ if (vs_ops->sched_tx_port)
+ vs_port = vs_ops->sched_tx_port(vs_dev, ptype, core_id);
+
+ return vs_port;
+}
+
+static int vs_check_mandatory_ops(struct vswitch_ops *ops)
+{
+ int rc = 0;
+
+ if (!ops->lookup_n_fwd){
+ RTE_LOG(ERR, VHOST_CONFIG, "lookup_n_fwd is NULL");
+ rc = -EINVAL;
+ }
+ if (!ops->learn_port){
+ RTE_LOG(ERR, VHOST_CONFIG, "learn_port is NULL");
+ rc = -EINVAL;
+ }
+
+ if (!ops->unlearn_port){
+ RTE_LOG(ERR, VHOST_CONFIG, "unlearn_port is NULL");
+ rc = -EINVAL;
+ }
+
+
+ return rc;
+}
+
+struct vswitch_dev *vs_register_switch(const char *name, int priv_size,
+ int max_ports, struct vswitch_ops *ops)
+{
+ struct vswitch_dev *vs_dev;
+ struct vswitch_port *vs_port;
+ int size, i;
+
+ if (vs_check_mandatory_ops(ops)) {
+ RTE_LOG(ERR, VHOST_CONFIG, "%s: mandatory ops missing\n",
+ name);
+ return NULL;
+ }
+
+ size = priv_size + sizeof(struct vswitch_dev);
+ vs_dev = rte_malloc(NULL, size, 64);
+ if (!vs_dev) {
+ RTE_LOG(DEBUG, VHOST_CONFIG, "Failed to vswitch device\n");
+ goto out;
+ }
+
+ strncpy(vs_dev->name, name, VSWITCH_NAME_SIZE);
+ vs_dev->priv = (void *) (vs_dev + 1);
+
+ size = max_ports * sizeof(struct vswitch_port);
+ vs_dev->ports = rte_malloc(NULL, size, 64);
+ if (!vs_dev->ports) {
+ RTE_LOG(DEBUG, VHOST_CONFIG,
+ "Failed allocate %d ports for vswitch %s\n",
+ max_ports, vs_dev->name);
+ goto out;
+ }
+
+ memset(vs_dev->ports, 0, size);
+ for (i = 0; i < max_ports; i++)
+ {
+ vs_port = &vs_dev->ports[i];
+ vs_port->state = VSWITCH_PSTATE_NOT_INUSE;
+ vs_port->vs_dev = vs_dev;
+ }
+
+ vs_dev->port_count = max_ports;
+ vs_dev->ops = ops;
+ rte_spinlock_init(&vs_dev->lock);
+
+ rte_spinlock_lock(&vs_mdata_g->lock);
+ LIST_INSERT_HEAD(&vs_mdata_g->head, vs_dev, list);
+ vs_mdata_g->dev_count++;
+ rte_spinlock_unlock(&vs_mdata_g->lock);
+
+ RTE_LOG(INFO, VHOST_CONFIG, "Added vswitch %s [0x%p], max_ports %d\n",
+ vs_dev->name, vs_dev, vs_dev->port_count);
+
+ return vs_dev;
+
+out:
+ if (vs_dev && vs_dev->ports)
+ rte_free(vs_dev->ports);
+ if (vs_dev)
+ rte_free(vs_dev);
+
+ return NULL;
+}
+
+int vs_unregister_switch(struct vswitch_dev *vs_dev)
+{
+ struct vswitch_dev *temp;
+ int removed = 0, rc;
+
+ rte_spinlock_lock(&vs_mdata_g->lock);
+ LIST_FOREACH(temp, &vs_mdata_g->head, list) {
+ if (!strncmp(temp->name, vs_dev->name, VSWITCH_NAME_SIZE)){
+ LIST_REMOVE(temp, list);
+ removed = 1;
+ }
+ }
+ rte_spinlock_unlock(&vs_mdata_g->lock);
+
+ if (!removed) {
+ RTE_LOG(DEBUG, VHOST_CONFIG, "vswitch [%s] not found\n",
+ vs_dev->name);
+ rc = -ENODEV;
+ }
+
+ RTE_LOG(DEBUG, VHOST_CONFIG, "Unregistering and freeing vswitch [%s]\n",
+ vs_dev->name);
+
+ if (vs_dev->ports)
+ rte_free(vs_dev->ports);
+
+ rte_free(vs_dev);
+
+ return rc;
+}
+
+int vs_get_max_vdevs(struct vswitch_dev *vs_dev)
+{
+ struct vswitch_ops *ops = vs_dev->ops;
+
+ return ops->get_max_vdevs(vs_dev);
+}
+
+int vs_switch_dev_init(struct vswitch_dev *vs_dev, uint16_t conf_flags)
+{
+ struct vswitch_ops *ops = vs_dev->ops;
+ int rc = 0;
+
+ if (ops->switch_init)
+ rc = ops->switch_init(vs_dev, conf_flags);
+
+ if (!rc)
+ vs_dev->conf_flags = conf_flags;
+
+ return rc;
+}
+
+int vs_vswitch_init(void)
+{
+ int rc = 0;
+
+ vs_mdata_g = rte_malloc(NULL, sizeof(struct vs_mdata), 64);
+ if (!vs_mdata_g)
+ {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "vhost vswitch mdata allocation failed\n");
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ memset(vs_mdata_g, 0, sizeof(struct vs_mdata));
+ LIST_INIT(&vs_mdata_g->head);
+ rte_spinlock_init(&vs_mdata_g->lock);
+
+out:
+ return rc;
+}
diff --git a/examples/vhost/vswitch_common.h b/examples/vhost/vswitch_common.h
new file mode 100644
index 0000000..a5800df
--- /dev/null
+++ b/examples/vhost/vswitch_common.h
@@ -0,0 +1,186 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2016 Freescale Semiconductor. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Freescale Semiconductor nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __VHOST_SWITCH_COMMON_H__
+#define __VHOST_SWITCH_COMMON_H__
+
+#include <sys/queue.h>
+#include "main.h"
+
+#define VSWITCH_NAME_SIZE 32
+
+enum vswitch_port_type {
+ VSWITCH_PTYPE_PHYS = 1,
+ VSWITCH_PTYPE_VIRTIO,
+ VSWITCH_PTYPE_END,
+};
+
+#define VS_PTYPE_MASK(type) (1 << type)
+
+enum vswitch_port_state {
+ VSWITCH_PSTATE_ADDED = 1,
+ VSWITCH_PSTATE_LEARNING,
+ VSWITCH_PSTATE_FORWARDING,
+ VSWITCH_PSTATE_NOT_INUSE,
+ VSWITCH_PSTATE_BEING_ALLOCATED,
+ VSWITCH_PSTATE_END,
+};
+
+struct vswitch_port {
+ enum vswitch_port_type type;
+ enum vswitch_port_state state;
+ unsigned int port_id;
+ struct rte_eth_conf port_conf;
+ struct vswitch_dev *vs_dev; /*switch to which this port belongs */
+ void *priv; /*Private for port specefic data*/
+ struct ether_addr mac_addr;
+ int phys_port_rxq;
+ uint16_t (*do_tx) (struct vswitch_port *port, uint16_t tx_q,
+ __attribute__((unused))struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **tx_pkts, uint16_t pkt_count);
+ uint16_t (*do_rx) (struct vswitch_port *port, uint16_t rx_q,
+ __attribute__((unused))struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **rx_pkts, uint16_t pkt_count);
+ uint16_t (*get_rxq) (struct vswitch_port *port, struct vhost_dev *vdev,
+ uint16_t core_id);
+ uint16_t (*get_txq) (struct vswitch_port *port, uint16_t core_id);
+};
+
+struct vswitch_dev {
+ char name[VSWITCH_NAME_SIZE];
+ void *priv; /* Private for switch specefic
+ * data */
+ uint64_t conf_flags;
+ rte_spinlock_t lock;
+ LIST_ENTRY (vswitch_dev) list;
+ struct vswitch_port *ports;
+ int port_count;
+ struct vswitch_ops *ops;
+};
+
+/* Function typedefs */
+typedef int (*vs_switch_init_t) (struct vswitch_dev *, uint64_t conf_flags);
+typedef int (*vs_add_port_t) (struct vswitch_port *vs_port);
+typedef int (*vs_del_port_t) (struct vswitch_port *port);
+typedef int (*vs_learn_port_t) (struct vswitch_port *port, struct rte_mbuf
+ **pkts, uint16_t count);
+typedef int (*vs_unlearn_port_t) (struct vswitch_port *port);
+typedef int (*vs_lookup_n_fwd_t)(struct vswitch_port *in_port, struct rte_mbuf
+ **pkts, uint16_t pkt_count, uint16_t rxq);
+#if 0
+typedef uint16_t (*vs_do_tx_t) (struct vswitch_port *port, uint16_t tx_q, struct
+ rte_mbuf **tx_pkts, uint16_t pkt_count);
+
+typedef uint16_t (*vs_do_rx_t) (struct vswitch_port *port, uint16_t rx_q, struct
+ rte_mbuf **rx_pkts, uint16_t pkt_count);
+#endif
+typedef struct vswitch_port* (*vs_sched_tx_port_t)(struct vswitch_dev *vs_dev,
+ enum vswitch_port_type ptype, uint16_t core_id);
+
+typedef struct vswitch_port* (*vs_sched_rx_port_t)(struct vswitch_dev *vs_dev,
+ enum vswitch_port_type ptype, uint16_t core_id);
+
+typedef int (*vs_port_start_t) (struct vswitch_port *port);
+typedef int (*vs_get_max_vdevs_t) (struct vswitch_dev *);
+
+struct vswitch_ops {
+ vs_add_port_t add_port;
+ vs_del_port_t del_port;
+ vs_lookup_n_fwd_t lookup_n_fwd;
+ vs_learn_port_t learn_port;
+ vs_unlearn_port_t unlearn_port;
+ vs_port_start_t port_start;
+ vs_port_start_t port_stop;
+ vs_switch_init_t switch_init;
+ vs_sched_tx_port_t sched_tx_port;
+ vs_sched_rx_port_t sched_rx_port;
+ vs_get_max_vdevs_t get_max_vdevs;
+};
+
+/*VSWITCH conf flags */
+#define VS_CNF_FLG_VM2VM_HARDWARE (1 << 0)
+#define VS_CNF_FLG_VM2VM_SOFTWARE (1 << 1)
+#define VS_CNF_FLG_PROMISCOUS_EN (1 << 2)
+#define VS_CNF_FLG_JUMBO_EN (1 << 3)
+#define VS_CNF_FLG_VLAN_STRIP_EN (1 << 4)
+#define VS_CNF_FLG_STATS_EN (1 << 5)
+
+/*API for vswitch implementations*/
+struct vswitch_dev *vs_register_switch(const char *name, int priv_size,
+ int max_ports, struct vswitch_ops *ops);
+int vs_unregister_switch(struct vswitch_dev *dev);
+
+int vs_vswitch_init(void);
+
+
+/*API for user of vswitch like vhost-switch application */
+
+struct vswitch_dev *vs_get_vswitch_dev(const char *name);
+struct vswitch_port *vs_add_port(struct vswitch_dev *vs_dev, int port_id,
+ enum vswitch_port_type type, void *priv);
+
+int vs_del_port(struct vswitch_port *port);
+
+int vs_switch_dev_init(struct vswitch_dev *vs_dev, uint16_t conf_flags);
+
+int vs_port_start(struct vswitch_port *vs_port);
+int vs_port_stop(struct vswitch_port *vs_port);
+int vs_learn_port(struct vswitch_port *port, struct rte_mbuf
+ **pkts, uint16_t count);
+int vs_unlearn_port(struct vswitch_port *port);
+int vs_lookup_n_fwd(struct vswitch_port *in_port, struct rte_mbuf
+ **pkts, uint16_t count, uint16_t in_rxq);
+
+int vs_do_broadcast_fwd(struct vswitch_dev *vs_dev,
+ struct vswitch_port *in_port, uint32_t txport_type_mask,
+ struct rte_mbuf *mbuf);
+
+struct vswitch_port *vs_sched_tx_port(struct vswitch_dev *vs_dev, enum
+ vswitch_port_type ptype, uint16_t core_id);
+
+struct vswitch_port *vs_sched_rx_port(struct vswitch_dev *vs_dev, enum
+ vswitch_port_type ptype, uint16_t core_id);
+int vs_get_max_vdevs(struct vswitch_dev *vs_dev);
+
+/*Extern APIs from vhost/main.c */
+
+extern struct mbuf_table *vhost_switch_get_txq(uint16_t core_id);
+extern int virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m);
+extern struct vhost_dev *find_vhost_dev(struct ether_addr *mac);
+void do_drain_mbuf_table(struct mbuf_table *tx_q);
+
+/* TBD:XXX: This needs to be removed here, when constructor mechanism
+ * for registering swittches is in place
+ */
+extern void vmdq_switch_impl_init(void);
+#endif
diff --git a/examples/vhost/vswitch_txrx.c b/examples/vhost/vswitch_txrx.c
new file mode 100644
index 0000000..23b38b9
--- /dev/null
+++ b/examples/vhost/vswitch_txrx.c
@@ -0,0 +1,97 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2016 Freescale Semiconductor. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Freescale Semiconductor nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <unistd.h>
+#include <rte_ethdev.h>
+#include <rte_log.h>
+#include <rte_virtio_net.h>
+
+#include "vswitch_common.h"
+#include "vswitch_txrx.h"
+
+uint16_t vs_do_tx_phys_port(struct vswitch_port *port, uint16_t tx_q,
+ __attribute__((unused))struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **pkts, uint16_t pkt_count)
+{
+ return rte_eth_tx_burst(port->port_id, tx_q, pkts, pkt_count);
+}
+
+
+uint16_t vs_do_tx_virtio_port(struct vswitch_port *port, uint16_t tx_q,
+ __attribute__((unused)) struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **pkts, uint16_t pkt_count)
+{
+ return rte_vhost_enqueue_burst(port->port_id, tx_q, pkts, pkt_count);
+}
+
+uint16_t vs_do_rx_phys_port (struct vswitch_port *port, uint16_t rx_q,
+ __attribute__((unused))struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **rx_pkts, uint16_t pkt_count)
+{
+ return rte_eth_rx_burst(port->port_id, rx_q, rx_pkts, pkt_count);
+}
+
+uint16_t vs_do_rx_virtio_port (struct vswitch_port *port, uint16_t rx_q,
+ struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **rx_pkts, uint16_t pkt_count)
+{
+ return rte_vhost_dequeue_burst(port->port_id, rx_q, mbuf_pool,
+ rx_pkts, pkt_count);
+}
+
+uint16_t vs_get_rxq_phys_port(__attribute__((unused)) struct vswitch_port *port,
+ struct vhost_dev *vdev, __attribute__((unused))uint16_t core_id)
+{
+ struct vswitch_port *vdev_vs_port = vdev->vs_port;
+
+ return vdev_vs_port->phys_port_rxq;
+}
+
+uint16_t vs_get_txq_phys_port(__attribute__((unused)) struct vswitch_port *port,
+ uint16_t core_id)
+{
+ return core_id;
+}
+
+uint16_t vs_get_rxq_virtio_port(__attribute__((unused)) struct vswitch_port *port,
+ __attribute__((unused))struct vhost_dev *vdev,
+ __attribute__((unused)) uint16_t core_id)
+{
+ return VIRTIO_TXQ;
+}
+
+uint16_t vs_get_txq_virtio_port(__attribute__((unused))struct vswitch_port *port,
+ __attribute__((unused)) uint16_t core_id)
+{
+ return VIRTIO_RXQ;
+}
diff --git a/examples/vhost/vswitch_txrx.h b/examples/vhost/vswitch_txrx.h
new file mode 100644
index 0000000..39d2dab
--- /dev/null
+++ b/examples/vhost/vswitch_txrx.h
@@ -0,0 +1,71 @@
+
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2016 Freescale Semiconductor. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Freescale Semiconductor nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef __VSWTICH_TXRX_H__
+#define __VSWTICH_TXRX_H__
+
+#include <unistd.h>
+#include <rte_ethdev.h>
+
+#include "vswitch_common.h"
+#include "vswitch_txrx.h"
+
+uint16_t vs_do_tx_phys_port(struct vswitch_port *port, uint16_t tx_q,
+ __attribute__((unused))struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **pkts, uint16_t pkt_count);
+
+
+uint16_t vs_do_tx_virtio_port(struct vswitch_port *port, uint16_t tx_q,
+ __attribute__((unused)) struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **pkts, uint16_t pkt_count);
+
+uint16_t vs_do_rx_phys_port (struct vswitch_port *port, uint16_t rx_q,
+ __attribute__((unused))struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **rx_pkts, uint16_t pkt_count);
+
+uint16_t vs_do_rx_virtio_port (struct vswitch_port *port, uint16_t rx_q,
+ struct rte_mempool *mbuf_pool,
+ struct rte_mbuf **rx_pkts, uint16_t pkt_count);
+
+uint16_t vs_get_rxq_phys_port(struct vswitch_port *port,
+ struct vhost_dev *vdev, uint16_t core_id);
+
+uint16_t vs_get_txq_phys_port(struct vswitch_port *port,
+ uint16_t core_id);
+
+uint16_t vs_get_rxq_virtio_port(struct vswitch_port *port,
+ struct vhost_dev *vdev, uint16_t core_id);
+
+uint16_t vs_get_txq_virtio_port(struct vswitch_port *port,
+ uint16_t core_id);
+#endif
--
1.9.1
More information about the dev
mailing list