[PATCH v1 2/3] raw/ntb: add AMD NTB support

[Bruce Richardson]

On Sun, Aug 23, 2026 at 07:36:38PM +0530, Raghavendra Ningoji wrote:
> Add support for the NTB endpoints integrated in AMD EPYC Embedded
> "Turin", "Genoa" and "Siena" processors to the raw/ntb driver.
> 
> The AMD NTB uses a primary/secondary topology: one endpoint enumerates
> as the primary (device ID 0x14c0) and the other as the secondary
> (device ID 0x14c3). The hardware exposes two memory windows (BAR23 and
> BAR45), 16 doorbells and a single shared 16-register scratchpad bank.
> The scratchpad bank is split into two disjoint 8-register sets, one per
> side, so the driver uses a packed handshake layout that fits in 8
> registers, plugged in through the framework's dev_handshake and
> read_peer_config hooks. A vendor-specific MSI-X interrupt handler is
> provided through the interrupt_handler hook.
> 
> AMD NTB uses an outbound translation window: a write to a BARxx memory
> window offset is forwarded to (xlat_base | offset) in the peer's memory
> rather than (xlat_base + offset). For that to be correct the
> translation base must be aligned to a power of two >= the window length
> so that no offset bit collides with a set bit in the base; the XLAT
> register also requires at least 4K alignment. Report this requirement
> to applications through a new mw_addr_align field in struct
> ntb_dev_info, reject a misaligned base in amd_ntb_mw_set_trans, and
> honour the field when reserving the memzone in the ntb example.
> 
> On the secondary side the device's own PCIe link status does not
> reflect the true inter-host link, so the link speed and width are read
> from the upstream switch port via sysfs.
> 
> Signed-off-by: Raghavendra Ningoji <raghavendra.ningoji at amd.com>
> ---
>  drivers/raw/ntb/meson.build   |   3 +-
>  drivers/raw/ntb/ntb.c         |  21 +
>  drivers/raw/ntb/ntb_hw_amd.c  | 709 ++++++++++++++++++++++++++++++++++
>  drivers/raw/ntb/ntb_hw_amd.h  | 114 ++++++
>  drivers/raw/ntb/rte_pmd_ntb.h |  10 +
>  examples/ntb/ntb_fwd.c        |  22 +-
>  usertools/dpdk-devbind.py     |   4 +-
>  7 files changed, 879 insertions(+), 4 deletions(-)
>  create mode 100644 drivers/raw/ntb/ntb_hw_amd.c
>  create mode 100644 drivers/raw/ntb/ntb_hw_amd.h
> 

Reviewing changes to common code only, please see inline below. I think
there is quite a bit of complexity introduced by the alignment constraints
which could do with being simplified.

Thanks,
/Bruce

> diff --git a/drivers/raw/ntb/meson.build b/drivers/raw/ntb/meson.build
> index 9096f2b25a..d7a8f2d1ed 100644
> --- a/drivers/raw/ntb/meson.build
> +++ b/drivers/raw/ntb/meson.build
> @@ -3,5 +3,6 @@
>  
>  deps += ['rawdev', 'mbuf', 'mempool', 'pci', 'bus_pci']
>  sources = files('ntb.c',
> -                'ntb_hw_intel.c')
> +                'ntb_hw_intel.c',
> +                'ntb_hw_amd.c')

Very minor nit, but consider putting the ")" on the next line and putting a
comma after the 'ntb_hw_amd.c' entry (since meson allows a trailing comma).
This means that new entries can be added without having to modify any
existing lines.

>  headers = files('rte_pmd_ntb.h')
> diff --git a/drivers/raw/ntb/ntb.c b/drivers/raw/ntb/ntb.c
> index 3a6a299081..b87141e4f4 100644
> --- a/drivers/raw/ntb/ntb.c
> +++ b/drivers/raw/ntb/ntb.c
> @@ -20,12 +20,15 @@
>  #include <rte_rawdev_pmd.h>
>  
>  #include "ntb_hw_intel.h"
> +#include "ntb_hw_amd.h"
>  #include "rte_pmd_ntb.h"
>  #include "ntb.h"
>  
>  static const struct rte_pci_id pci_id_ntb_map[] = {
>  	{ RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_SKX) },
>  	{ RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_ICX) },
> +	{ RTE_PCI_DEVICE(NTB_AMD_VENDOR_ID, NTB_AMD_DEV_ID_PRI) },
> +	{ RTE_PCI_DEVICE(NTB_AMD_VENDOR_ID, NTB_AMD_DEV_ID_SEC) },
>  	{ .vendor_id = 0, /* sentinel */ },
>  };
>  
> @@ -846,6 +849,20 @@ ntb_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info,
>  	info->mw_size_align = (uint8_t)(hw->pci_dev->id.vendor_id ==
>  					NTB_INTEL_VENDOR_ID);
>  
> +	/**
> +	 * AMD NTB uses an outbound translation window: writes to a BARxx
> +	 * memory window are forwarded to the peer via the XLAT registers,
> +	 * whose base must be 4K aligned. If the mw memzone base is not
> +	 * aligned, the low bits are dropped and all window writes land at
> +	 * the wrong offset. Report the required alignment so the memzone is
> +	 * reserved correctly. Intel uses mw_size_align (a superset), so this
> +	 * only matters for non-Intel vendors.
> +	 */

Consider shortening the coment here. Also see other feedback below
regarding this field.

> +	if (hw->pci_dev->id.vendor_id == NTB_AMD_VENDOR_ID)
> +		info->mw_addr_align = RTE_PGSIZE_4K;
> +	else
> +		info->mw_addr_align = 0;
> +
>  	if (!hw->queue_size || !hw->queue_pairs) {
>  		NTB_LOG(ERR, "No queue size and queue num assigned.");
>  		return -EAGAIN;
> @@ -1406,6 +1423,10 @@ ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev)
>  	case NTB_INTEL_DEV_ID_B2B_ICX:
>  		hw->ntb_ops = &intel_ntb_ops;
>  		break;
> +	case NTB_AMD_DEV_ID_PRI:
> +	case NTB_AMD_DEV_ID_SEC:
> +		hw->ntb_ops = &amd_ntb_ops;
> +		break;
>  	default:
>  		NTB_LOG(ERR, "Not supported device.");
>  		return -EINVAL;
> diff --git a/drivers/raw/ntb/ntb_hw_amd.c b/drivers/raw/ntb/ntb_hw_amd.c
> new file mode 100644
> index 0000000000..9861dcee57
> --- /dev/null
> +++ b/drivers/raw/ntb/ntb_hw_amd.c

<snip>

> diff --git a/drivers/raw/ntb/rte_pmd_ntb.h b/drivers/raw/ntb/rte_pmd_ntb.h
> index 76da3be026..59a2ad6849 100644
> --- a/drivers/raw/ntb/rte_pmd_ntb.h
> +++ b/drivers/raw/ntb/rte_pmd_ntb.h
> @@ -27,6 +27,16 @@ struct ntb_dev_info {
>  	uint8_t mw_size_align;
>  	uint8_t mw_cnt;
>  	uint64_t *mw_size;
> +	/**< Minimum alignment (bytes) required for the mw translation base
> +	 * address, and a flag that the base must additionally be aligned to a
> +	 * power of two >= the mw length. 0 means no extra alignment beyond
> +	 * cache line. AMD NTB uses an outbound translation window that forms
> +	 * the target as (xlat_base | offset) instead of (xlat_base + offset),
> +	 * so the base must be size-aligned to avoid offset bits colliding with
> +	 * base bits; it also requires at least 4K alignment for the XLAT
> +	 * register.
> +	 */
> +	uint64_t mw_addr_align;

Again, shorten the comment here to just a line or two.
Also, in terms of how it is used, remove the special case for 0 == cache
aligned, and instead change code assignment above to be cache aligned by
default. This means that all uses of this value in apps don't need to have a
special-case for it - they just align the memory allocation to what is
provided, be it 64-bytes or 4k.

>  };
>  
>  struct ntb_dev_config {
> diff --git a/examples/ntb/ntb_fwd.c b/examples/ntb/ntb_fwd.c
> index 33f3c1ef17..7cc4e22147 100644
> --- a/examples/ntb/ntb_fwd.c
> +++ b/examples/ntb/ntb_fwd.c
> @@ -1146,8 +1146,10 @@ ntb_mbuf_pool_create(uint16_t mbuf_seg_size, uint32_t nb_mbuf,
>  		if (!left_sz)
>  			break;
>  		snprintf(mz_name, sizeof(mz_name), "ntb_mw_%d", mz_id);
> -		align = ntb_info.mw_size_align ? ntb_info.mw_size[mz_id] :
> -			RTE_CACHE_LINE_SIZE;
> +		if (ntb_info.mw_size_align)
> +			align = ntb_info.mw_size[mz_id];
> +		else
> +			align = RTE_CACHE_LINE_SIZE;

Again, see above comment. If we remove zero as a possible value here, you
can just use ntb_info.mw_size_align directly without branching in app code.

>  		/* Reserve ntb header space on memzone 0. */
>  		max_mz_len = mz_id ? ntb_info.mw_size[mz_id] :
>  			     ntb_info.mw_size[mz_id] - ntb_info.ntb_hdr_size;
> @@ -1155,6 +1157,22 @@ ntb_mbuf_pool_create(uint16_t mbuf_seg_size, uint32_t nb_mbuf,
>  			(max_mz_len / total_elt_sz * total_elt_sz);
>  		if (!mz_len)
>  			continue;
> +		/*
> +		 * Some NTB hardware (e.g. AMD) uses an outbound translation
> +		 * window that forms the target as (xlat_base | offset) rather
> +		 * than (xlat_base + offset). For that to be correct the memzone
> +		 * base must be aligned to a power of two >= its length, so that
> +		 * no offset bit collides with a set bit in the base address.
> +		 * Honour that requirement when the driver reports mw_addr_align.
> +		 */
> +		if (ntb_info.mw_addr_align) {
> +			uint64_t pow2_align = rte_align64pow2(mz_len);
> +
> +			if (pow2_align > align)
> +				align = pow2_align;
> +			if (ntb_info.mw_addr_align > align)
> +				align = ntb_info.mw_addr_align;
> +		}

This is really complex to have in the app, and is hard for the user to
understand and work with too. I would suggest that, rather than trying to
expose this via a single addr_align value - which it turns out isn't
actually the alignment needed - you add a separate API called
"get_mem_align" or something similar, and then hide the complexity of this
calculation in the driver. Then you can drop the mw_addr_align in the info
struct.

>  		mz = rte_memzone_reserve_aligned(mz_name, mz_len, socket_id,
>  					RTE_MEMZONE_IOVA_CONTIG, align);
>  		if (mz == NULL) {
> diff --git a/usertools/dpdk-devbind.py b/usertools/dpdk-devbind.py
> index e72f238aba..cf5747b003 100755
> --- a/usertools/dpdk-devbind.py
> +++ b/usertools/dpdk-devbind.py
> @@ -78,6 +78,8 @@
>                   'SVendor': None, 'SDevice': None}
>  intel_ntb_icx = {'Class': '06', 'Vendor': '8086', 'Device': '347e',
>                   'SVendor': None, 'SDevice': None}
> +amd_ntb = {'Class': '06', 'Vendor': '1022', 'Device': '14c0,14c3',
> +           'SVendor': None, 'SDevice': None}
>  
>  cnxk_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a0f9,a0fa',
>              'SVendor': None, 'SDevice': None}
> @@ -105,7 +107,7 @@
>  regex_devices = [cn9k_ree]
>  ml_devices = [cnxk_ml]
>  misc_devices = [cnxk_bphy, cnxk_bphy_cgx, cnxk_inl_dev,
> -                intel_ntb_skx, intel_ntb_icx,
> +                intel_ntb_skx, intel_ntb_icx, amd_ntb,
>                  virtio_blk]
>  
>  # global dict ethernet devices present. Dictionary indexed by PCI address.
> -- 
> 2.34.1
>