[dpdk-dev] [PATCH v3 04/11] linuxapp/eal_pci: get iommu class

Sergio Gonzalez Monroy sergio.gonzalez.monroy at intel.com
Fri Jul 14 09:43:28 CEST 2017


On 13/07/2017 09:23, santosh wrote:
> Hi Sergio,
>
> On Wednesday 12 July 2017 01:50 PM, Sergio Gonzalez Monroy wrote:
>
>> On 11/07/2017 07:16, Santosh Shukla wrote:
>>> Get iommu class of PCI device on the bus and returns preferred iova
>>> mapping mode for that bus.
>>>
>>> Algorithm for iova scheme selection for PCI bus:
>>> 0. Look for device attached to vfio kdrv and has .drv_flag set
>>> to RTE_PCI_DRV_NEED_IOVA_VA.
>>> 1. Look for any device attached to UIO class of driver.
>>> 2. Check for vfio-noiommu mode enabled.
>>>
>>> If 1) & 2) is false and 0) is true then select
>>> mapping scheme as iova=va. Otherwise use default
>>> mapping scheme (iova_pa).
>>>
>>> Signed-off-by: Santosh Shukla <santosh.shukla at caviumnetworks.com>
>>> Signed-off-by: Jerin Jacob <jerin.jacob at caviumnetworks.com>
>>> ---
>>> v1 --> v2:
>>> - Removed Linux version check in vfio_noiommu func. Refer [1].
>>> - Extending autodetction logic for _iommu_class.
>>> Refer [2].
>>>
>>> [1] https://www.mail-archive.com/dev@dpdk.org/msg70108.html
>>> [2] https://www.mail-archive.com/dev@dpdk.org/msg70279.html
>> Just wondering how it all works with device hotplug.
>> Correct me if I am wrong but if EAL decides to use IOVA_AS_VA scheme,
>> if we were to attach a device that needs IOVA_AS_PA, it will not work and should fail to attach, right?
>>
> It will work for igb_uio case, and won't work for vfio-noiommu hotplug case(Invalid case).

Why are those two cases (igb_uio, vfio-noiommu) different? do they not 
have the same requirements, ie. need IOVA_PA sheme?

Thanks,
Sergio

> Yes, we can dictate iova awareness to hotplug/unplug area.
>
>> Thanks,
>> Sergio
>>
>>>    lib/librte_eal/linuxapp/eal/eal_pci.c           | 66 +++++++++++++++++++++++++
>>>    lib/librte_eal/linuxapp/eal/eal_vfio.c          | 19 +++++++
>>>    lib/librte_eal/linuxapp/eal/eal_vfio.h          |  4 ++
>>>    lib/librte_eal/linuxapp/eal/rte_eal_version.map |  1 +
>>>    4 files changed, 90 insertions(+)
>>>
>>> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
>>> index 7d9e1a99b..573caa000 100644
>>> --- a/lib/librte_eal/linuxapp/eal/eal_pci.c
>>> +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
>>> @@ -45,6 +45,7 @@
>>>    #include "eal_filesystem.h"
>>>    #include "eal_private.h"
>>>    #include "eal_pci_init.h"
>>> +#include "eal_vfio.h"
>>>      /**
>>>     * @file
>>> @@ -488,6 +489,71 @@ rte_pci_scan(void)
>>>        return -1;
>>>    }
>>>    +/*
>>> + * Any one of the device bound to uio
>>> + */
>>> +static inline int
>>> +pci_device_bound_uio(void)
>>> +{
>>> +    struct rte_pci_device *dev = NULL;
>>> +
>>> +    FOREACH_DEVICE_ON_PCIBUS(dev) {
>>> +        if (dev->kdrv == RTE_KDRV_IGB_UIO ||
>>> +           dev->kdrv == RTE_KDRV_UIO_GENERIC) {
>>> +            return 1;
>>> +        }
>>> +    }
>>> +    return 0;
>>> +}
>>> +
>>> +/*
>>> + * Any one of the device has iova as va
>>> + */
>>> +static inline int
>>> +pci_device_has_iova_va(void)
>>> +{
>>> +    struct rte_pci_device *dev = NULL;
>>> +    struct rte_pci_driver *drv = NULL;
>>> +
>>> +    FOREACH_DRIVER_ON_PCIBUS(drv) {
>>> +        if (drv && drv->drv_flags & RTE_PCI_DRV_NEED_IOVA_VA) {
>>> +            FOREACH_DEVICE_ON_PCIBUS(dev) {
>>> +                if (dev->kdrv == RTE_KDRV_VFIO &&
>>> +                    rte_pci_match(drv, dev))
>>> +                    return 1;
>>> +            }
>>> +        }
>>> +    }
>>> +    return 0;
>>> +}
>>> +
>>> +/*
>>> + * Get iommu class of PCI devices on the bus.
>>> + */
>>> +enum rte_iova_mode
>>> +rte_pci_get_iommu_class(void)
>>> +{
>>> +    bool is_vfio_noiommu_enabled;
>>> +    bool has_iova_va;
>>> +    bool is_bound_uio;
>>> +
>>> +    has_iova_va = pci_device_has_iova_va();
>>> +    is_bound_uio = pci_device_bound_uio();
>>> +    is_vfio_noiommu_enabled = vfio_noiommu_is_enabled() == 1 ? 1 : 0;
>>> +
>>> +    if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled)
>>> +        return RTE_IOVA_VA;
>>> +
>>> +    if (has_iova_va) {
>>> +        if (is_vfio_noiommu_enabled)
>>> +            RTE_LOG(WARNING, EAL, "vfio-noiommu mode configured\n");
>>> +        if (is_bound_uio)
>>> +            RTE_LOG(WARNING, EAL, "Some device attached to UIO\n");
>>> +    }
>>> +
>>> +    return RTE_IOVA_PA;
>>> +}
>>> +
>>>    /* Read PCI config space. */
>>>    int rte_pci_read_config(const struct rte_pci_device *device,
>>>            void *buf, size_t len, off_t offset)
>>> diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.c b/lib/librte_eal/linuxapp/eal/eal_vfio.c
>>> index 946df7e31..c8a97b7e7 100644
>>> --- a/lib/librte_eal/linuxapp/eal/eal_vfio.c
>>> +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.c
>>> @@ -816,4 +816,23 @@ vfio_noiommu_dma_map(int __rte_unused vfio_container_fd)
>>>        return 0;
>>>    }
>>>    +int
>>> +vfio_noiommu_is_enabled(void)
>>> +{
>>> +    int fd, ret, cnt __rte_unused;
>>> +    char c;
>>> +
>>> +    ret = -1;
>>> +    fd = open(VFIO_NOIOMMU_MODE, O_RDONLY);
>>> +    if (fd < 0)
>>> +        return -1;
>>> +
>>> +    cnt = read(fd, &c, 1);
>>> +    if (c == 'Y')
>>> +        ret = 1;
>>> +
>>> +    close(fd);
>>> +    return ret;
>>> +}
>>> +
>>>    #endif
>>> diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.h b/lib/librte_eal/linuxapp/eal/eal_vfio.h
>>> index 5ff63e5d7..26ea8e119 100644
>>> --- a/lib/librte_eal/linuxapp/eal/eal_vfio.h
>>> +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.h
>>> @@ -150,6 +150,8 @@ struct vfio_config {
>>>    #define VFIO_NOIOMMU_GROUP_FMT "/dev/vfio/noiommu-%u"
>>>    #define VFIO_GET_REGION_ADDR(x) ((uint64_t) x << 40ULL)
>>>    #define VFIO_GET_REGION_IDX(x) (x >> 40)
>>> +#define VFIO_NOIOMMU_MODE      \
>>> +    "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode"
>>>      /* DMA mapping function prototype.
>>>     * Takes VFIO container fd as a parameter.
>>> @@ -210,6 +212,8 @@ int pci_vfio_is_enabled(void);
>>>      int vfio_mp_sync_setup(void);
>>>    +int vfio_noiommu_is_enabled(void);
>>> +
>>>    #define SOCKET_REQ_CONTAINER 0x100
>>>    #define SOCKET_REQ_GROUP 0x200
>>>    #define SOCKET_CLR_GROUP 0x300
>>> diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
>>> index c91dd44c4..044f89c7c 100644
>>> --- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
>>> +++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
>>> @@ -206,6 +206,7 @@ DPDK_17.08 {
>>>        rte_bus_find_by_device;
>>>        rte_bus_find_by_name;
>>>        rte_pci_match;
>>> +    rte_pci_get_iommu_class;
>>>      } DPDK_17.05;
>>>    
>>



More information about the dev mailing list