<div dir="ltr"><div dir="ltr">On Mon, Nov 14, 2022 at 11:54 AM Juraj Linkeš <juraj.linkes@pantheon.tech> wrote:<br></div><div class="gmail_quote"><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">Add methods for setting up and shutting down DPDK apps and for<br>
constructing EAL parameters.<br>
<br>
Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech><br>
---<br>
dts/conf.yaml | 4 +<br>
dts/framework/config/__init__.py | 85 ++++++++-<br>
dts/framework/config/conf_yaml_schema.json | 22 +++<br>
.../remote_session/os/linux_session.py | 15 ++<br>
dts/framework/remote_session/os/os_session.py | 16 +-<br>
.../remote_session/os/posix_session.py | 80 ++++++++<br>
dts/framework/testbed_model/hw/__init__.py | 17 ++<br>
dts/framework/testbed_model/hw/cpu.py | 164 ++++++++++++++++<br>
dts/framework/testbed_model/node/node.py | 36 ++++<br>
dts/framework/testbed_model/node/sut_node.py | 178 +++++++++++++++++-<br>
dts/framework/utils.py | 20 ++<br>
11 files changed, 634 insertions(+), 3 deletions(-)<br>
create mode 100644 dts/framework/testbed_model/hw/__init__.py<br>
create mode 100644 dts/framework/testbed_model/hw/cpu.py<br>
<br>
diff --git a/dts/conf.yaml b/dts/conf.yaml<br>
index 6b0bc5c2bf..976888a88e 100644<br>
--- a/dts/conf.yaml<br>
+++ b/dts/conf.yaml<br>
@@ -12,4 +12,8 @@ nodes:<br>
- name: "SUT 1"<br>
hostname: sut1.change.me.localhost<br>
user: root<br>
+ arch: x86_64<br>
os: linux<br>
+ bypass_core0: true<br>
+ cpus: ""<br>
+ memory_channels: 4<br>
diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py<br>
index 1b97dc3ab9..344d697a69 100644<br>
--- a/dts/framework/config/__init__.py<br>
+++ b/dts/framework/config/__init__.py<br>
@@ -11,12 +11,13 @@<br>
import pathlib<br>
from dataclasses import dataclass<br>
from enum import Enum, auto, unique<br>
-from typing import Any<br>
+from typing import Any, Iterable<br>
<br>
import warlock # type: ignore<br>
import yaml<br>
<br>
from framework.settings import SETTINGS<br>
+from framework.utils import expand_range<br>
<br>
<br>
class StrEnum(Enum):<br>
@@ -60,6 +61,80 @@ class Compiler(StrEnum):<br>
msvc = auto()<br>
<br>
<br>
+@dataclass(slots=True, frozen=True)<br>
+class CPU:<br>
+ cpu: int<br>
+ core: int<br>
+ socket: int<br>
+ node: int<br>
+<br>
+ def __str__(self) -> str:<br>
+ return str(self.cpu)<br>
+<br>
+<br>
+class CPUList(object):<br>
+ """<br>
+ Convert these options into a list of int cpus<br>
+ cpu_list=[CPU1, CPU2] - a list of CPUs<br>
+ cpu_list=[0,1,2,3] - a list of int indices<br>
+ cpu_list=['0','1','2-3'] - a list of str indices; ranges are supported<br>
+ cpu_list='0,1,2-3' - a comma delimited str of indices; ranges are supported<br>
+<br>
+ The class creates a unified format used across the framework and allows<br>
+ the user to use either a str representation (using str(instance) or directly<br>
+ in f-strings) or a list representation (by accessing instance.cpu_list).<br>
+ Empty cpu_list is allowed.<br>
+ """<br>
+<br>
+ _cpu_list: list[int]<br>
+<br>
+ def __init__(self, cpu_list: list[int | str | CPU] | str):<br>
+ self._cpu_list = []<br>
+ if isinstance(cpu_list, str):<br>
+ self._from_str(cpu_list.split(","))<br>
+ else:<br>
+ self._from_str((str(cpu) for cpu in cpu_list))<br>
+<br>
+ # the input cpus may not be sorted<br>
+ self._cpu_list.sort()<br>
+<br>
+ @property<br>
+ def cpu_list(self) -> list[int]:<br>
+ return self._cpu_list<br>
+<br>
+ def _from_str(self, cpu_list: Iterable[str]) -> None:<br>
+ for cpu in cpu_list:<br>
+ self._cpu_list.extend(expand_range(cpu))<br>
+<br>
+ def _get_consecutive_cpus_range(self, cpu_list: list[int]) -> list[str]:<br>
+ formatted_core_list = []<br>
+ tmp_cpus_list = list(sorted(cpu_list))<br>
+ segment = tmp_cpus_list[:1]<br>
+ for core_id in tmp_cpus_list[1:]:<br>
+ if core_id - segment[-1] == 1:<br>
+ segment.append(core_id)<br>
+ else:<br>
+ formatted_core_list.append(<br>
+ f"{segment[0]}-{segment[-1]}"<br>
+ if len(segment) > 1<br>
+ else f"{segment[0]}"<br>
+ )<br>
+ current_core_index = tmp_cpus_list.index(core_id)<br>
+ formatted_core_list.extend(<br>
+ self._get_consecutive_cpus_range(tmp_cpus_list[current_core_index:])<br>
+ )<br>
+ segment.clear()<br>
+ break<br>
+ if len(segment) > 0:<br>
+ formatted_core_list.append(<br>
+ f"{segment[0]}-{segment[-1]}" if len(segment) > 1 else f"{segment[0]}"<br>
+ )<br>
+ return formatted_core_list<br>
+<br>
+ def __str__(self) -> str:<br>
+ return f'{",".join(self._get_consecutive_cpus_range(self._cpu_list))}'<br>
+<br>
+<br>
# Slots enables some optimizations, by pre-allocating space for the defined<br>
# attributes in the underlying data structure.<br>
#<br>
@@ -71,7 +146,11 @@ class NodeConfiguration:<br>
hostname: str<br>
user: str<br>
password: str | None<br>
+ arch: Architecture<br>
os: OS<br>
+ bypass_core0: bool<br>
+ cpus: CPUList<br>
+ memory_channels: int<br>
<br>
@staticmethod<br>
def from_dict(d: dict) -> "NodeConfiguration":<br>
@@ -80,7 +159,11 @@ def from_dict(d: dict) -> "NodeConfiguration":<br>
hostname=d["hostname"],<br>
user=d["user"],<br>
password=d.get("password"),<br>
+ arch=Architecture(d["arch"]),<br>
os=OS(d["os"]),<br>
+ bypass_core0=d.get("bypass_core0", False),<br>
+ cpus=CPUList(d.get("cpus", "1")),<br>
+ memory_channels=d.get("memory_channels", 1),<br>
)<br>
<br>
<br>
diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json<br>
index 409ce7ac74..c59d3e30e6 100644<br>
--- a/dts/framework/config/conf_yaml_schema.json<br>
+++ b/dts/framework/config/conf_yaml_schema.json<br>
@@ -6,6 +6,12 @@<br>
"type": "string",<br>
"description": "A unique identifier for a node"<br>
},<br>
+ "ARCH": {<br>
+ "type": "string",<br>
+ "enum": [<br>
+ "x86_64"<br></blockquote><div><br></div><div>arm64 and ppc64le should probably be included here. I think that we can focus on 64 bit arches for now. </div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+ ]<br>
+ },<br>
"OS": {<br>
"type": "string",<br>
"enum": [<br>
@@ -82,8 +88,23 @@<br>
"type": "string",<br>
"description": "The password to use on this node. Use only as a last resort. SSH keys are STRONGLY preferred."<br>
},<br>
+ "arch": {<br>
+ "$ref": "#/definitions/ARCH"<br>
+ },<br>
"os": {<br>
"$ref": "#/definitions/OS"<br>
+ },<br>
+ "bypass_core0": {<br>
+ "type": "boolean",<br>
+ "description": "Indicate that DPDK should omit using the first core."<br>
+ },<br>
+ "cpus": {<br>
+ "type": "string",<br>
+ "description": "Optional comma-separated list of cpus to use, e.g.: 1,2,3,4,5,18-22. Defaults to 1. An empty string means use all cpus."<br>
+ },<br>
+ "memory_channels": {<br>
+ "type": "integer",<br>
+ "description": "How many memory channels to use. Optional, defaults to 1."<br>
}<br>
},<br>
"additionalProperties": false,<br>
@@ -91,6 +112,7 @@<br>
"name",<br>
"hostname",<br>
"user",<br>
+ "arch",<br>
"os"<br>
]<br>
},<br>
diff --git a/dts/framework/remote_session/os/linux_session.py b/dts/framework/remote_session/os/linux_session.py<br>
index 39e80631dd..21f117b714 100644<br>
--- a/dts/framework/remote_session/os/linux_session.py<br>
+++ b/dts/framework/remote_session/os/linux_session.py<br>
@@ -2,6 +2,8 @@<br>
# Copyright(c) 2022 PANTHEON.tech s.r.o.<br>
# Copyright(c) 2022 University of New Hampshire<br>
<br>
+from framework.config import CPU<br>
+<br>
from .posix_session import PosixSession<br>
<br>
<br>
@@ -9,3 +11,16 @@ class LinuxSession(PosixSession):<br>
"""<br>
The implementation of non-Posix compliant parts of Linux remote sessions.<br>
"""<br>
+<br>
+ def get_remote_cpus(self, bypass_core0: bool) -> list[CPU]:<br>
+ cpu_info = self.remote_session.send_command(<br>
+ "lscpu -p=CPU,CORE,SOCKET,NODE|grep -v \\#"<br>
+ ).stdout<br>
+ cpus = []<br>
+ for cpu_line in cpu_info.splitlines():<br>
+ cpu, core, socket, node = cpu_line.split(",")<br>
+ if bypass_core0 and core == 0 and socket == 0:<br>
+ <a href="http://self.logger.info" rel="noreferrer" target="_blank">self.logger.info</a>("Core0 bypassed.")<br>
+ continue<br>
+ cpus.append(CPU(int(cpu), int(core), int(socket), int(node)))<br>
+ return cpus<br>
diff --git a/dts/framework/remote_session/os/os_session.py b/dts/framework/remote_session/os/os_session.py<br>
index 57e2865282..6f6b6a979e 100644<br>
--- a/dts/framework/remote_session/os/os_session.py<br>
+++ b/dts/framework/remote_session/os/os_session.py<br>
@@ -3,9 +3,10 @@<br>
# Copyright(c) 2022 University of New Hampshire<br>
<br>
from abc import ABC, abstractmethod<br>
+from collections.abc import Iterable<br>
from pathlib import PurePath<br>
<br>
-from framework.config import Architecture, NodeConfiguration<br>
+from framework.config import CPU, Architecture, NodeConfiguration<br>
from framework.logger import DTSLOG<br>
from framework.remote_session.factory import create_remote_session<br>
from framework.remote_session.remote_session import RemoteSession<br>
@@ -130,3 +131,16 @@ def get_dpdk_version(self, version_path: str | PurePath) -> str:<br>
"""<br>
Inspect DPDK version on the remote node from version_path.<br>
"""<br>
+<br>
+ @abstractmethod<br>
+ def get_remote_cpus(self, bypass_core0: bool) -> list[CPU]:<br>
+ """<br>
+ Compose a list of CPUs present on the remote node.<br>
+ """<br>
+<br>
+ @abstractmethod<br>
+ def kill_cleanup_dpdk_apps(self, dpdk_prefix_list: Iterable[str]) -> None:<br>
+ """<br>
+ Kill and cleanup all DPDK apps identified by dpdk_prefix_list. If<br>
+ dpdk_prefix_list is empty, attempt to find running DPDK apps to kill and clean.<br>
+ """<br>
diff --git a/dts/framework/remote_session/os/posix_session.py b/dts/framework/remote_session/os/posix_session.py<br>
index a36b8e8c1a..7151263c7a 100644<br>
--- a/dts/framework/remote_session/os/posix_session.py<br>
+++ b/dts/framework/remote_session/os/posix_session.py<br>
@@ -2,6 +2,8 @@<br>
# Copyright(c) 2022 PANTHEON.tech s.r.o.<br>
# Copyright(c) 2022 University of New Hampshire<br>
<br>
+import re<br>
+from collections.abc import Iterable<br>
from pathlib import PurePath, PurePosixPath<br>
<br>
from framework.config import Architecture<br>
@@ -138,3 +140,81 @@ def get_dpdk_version(self, build_dir: str | PurePath) -> str:<br>
f"cat {self.join_remote_path(build_dir, 'VERSION')}", verify=True<br>
)<br>
return out.stdout<br>
+<br>
+ def kill_cleanup_dpdk_apps(self, dpdk_prefix_list: Iterable[str]) -> None:<br>
+ <a href="http://self.logger.info" rel="noreferrer" target="_blank">self.logger.info</a>("Cleaning up DPDK apps.")<br>
+ dpdk_runtime_dirs = self._get_dpdk_runtime_dirs(dpdk_prefix_list)<br>
+ if dpdk_runtime_dirs:<br>
+ # kill and cleanup only if DPDK is running<br>
+ dpdk_pids = self._get_dpdk_pids(dpdk_runtime_dirs)<br>
+ for dpdk_pid in dpdk_pids:<br>
+ self.remote_session.send_command(f"kill -9 {dpdk_pid}", 20)<br>
+ self._check_dpdk_hugepages(dpdk_runtime_dirs)<br>
+ self._remove_dpdk_runtime_dirs(dpdk_runtime_dirs)<br>
+<br>
+ def _get_dpdk_runtime_dirs(<br>
+ self, dpdk_prefix_list: Iterable[str]<br>
+ ) -> list[PurePosixPath]:<br>
+ prefix = PurePosixPath("/var", "run", "dpdk")<br>
+ if not dpdk_prefix_list:<br>
+ remote_prefixes = self._list_remote_dirs(prefix)<br>
+ if not remote_prefixes:<br>
+ dpdk_prefix_list = []<br>
+ else:<br>
+ dpdk_prefix_list = remote_prefixes<br>
+<br>
+ return [PurePosixPath(prefix, dpdk_prefix) for dpdk_prefix in dpdk_prefix_list]<br>
+<br>
+ def _list_remote_dirs(self, remote_path: str | PurePath) -> list[str] | None:<br>
+ """<br>
+ Return a list of directories of the remote_dir.<br>
+ If remote_path doesn't exist, return None.<br>
+ """<br>
+ out = self.remote_session.send_command(<br>
+ f"ls -l {remote_path} | awk '/^d/ {{print $NF}}'"<br>
+ ).stdout<br>
+ if "No such file or directory" in out:<br>
+ return None<br>
+ else:<br>
+ return out.splitlines()<br>
+<br>
+ def _get_dpdk_pids(self, dpdk_runtime_dirs: Iterable[str | PurePath]) -> list[int]:<br>
+ pids = []<br>
+ pid_regex = r"p(\d+)"<br>
+ for dpdk_runtime_dir in dpdk_runtime_dirs:<br>
+ dpdk_config_file = PurePosixPath(dpdk_runtime_dir, "config")<br>
+ if self._remote_files_exists(dpdk_config_file):<br>
+ out = self.remote_session.send_command(<br>
+ f"lsof -Fp {dpdk_config_file}"<br>
+ ).stdout<br>
+ if out and "No such file or directory" not in out:<br>
+ for out_line in out.splitlines():<br>
+ match = re.match(pid_regex, out_line)<br>
+ if match:<br>
+ pids.append(int(match.group(1)))<br>
+ return pids<br>
+<br>
+ def _remote_files_exists(self, remote_path: PurePath) -> bool:<br>
+ result = self.remote_session.send_command(f"test -e {remote_path}")<br>
+ return not result.return_code<br>
+<br>
+ def _check_dpdk_hugepages(<br>
+ self, dpdk_runtime_dirs: Iterable[str | PurePath]<br>
+ ) -> None:<br>
+ for dpdk_runtime_dir in dpdk_runtime_dirs:<br>
+ hugepage_info = PurePosixPath(dpdk_runtime_dir, "hugepage_info")<br>
+ if self._remote_files_exists(hugepage_info):<br>
+ out = self.remote_session.send_command(<br>
+ f"lsof -Fp {hugepage_info}"<br>
+ ).stdout<br>
+ if out and "No such file or directory" not in out:<br>
+ self.logger.warning("Some DPDK processes did not free hugepages.")<br>
+ self.logger.warning("*******************************************")<br>
+ self.logger.warning(out)<br>
+ self.logger.warning("*******************************************")<br>
+<br>
+ def _remove_dpdk_runtime_dirs(<br>
+ self, dpdk_runtime_dirs: Iterable[str | PurePath]<br>
+ ) -> None:<br>
+ for dpdk_runtime_dir in dpdk_runtime_dirs:<br>
+ self.remove_remote_dir(dpdk_runtime_dir)<br>
diff --git a/dts/framework/testbed_model/hw/__init__.py b/dts/framework/testbed_model/hw/__init__.py<br>
new file mode 100644<br>
index 0000000000..7d79a7efd0<br>
--- /dev/null<br>
+++ b/dts/framework/testbed_model/hw/__init__.py<br>
@@ -0,0 +1,17 @@<br>
+# SPDX-License-Identifier: BSD-3-Clause<br>
+# Copyright(c) 2022 PANTHEON.tech s.r.o.<br>
+<br>
+from framework.config import CPU, CPUList<br>
+<br>
+from .cpu import CPUAmount, CPUAmountFilter, CPUFilter, CPUListFilter<br>
+<br>
+<br>
+def cpu_filter(<br>
+ core_list: list[CPU], filter_specifier: CPUAmount | CPUList, ascending: bool<br>
+) -> CPUFilter:<br>
+ if isinstance(filter_specifier, CPUList):<br>
+ return CPUListFilter(core_list, filter_specifier, ascending)<br>
+ elif isinstance(filter_specifier, CPUAmount):<br>
+ return CPUAmountFilter(core_list, filter_specifier, ascending)<br>
+ else:<br>
+ raise ValueError(f"Unsupported filter r{filter_specifier}")<br>
diff --git a/dts/framework/testbed_model/hw/cpu.py b/dts/framework/testbed_model/hw/cpu.py<br>
new file mode 100644<br>
index 0000000000..87e87bcb4e<br>
--- /dev/null<br>
+++ b/dts/framework/testbed_model/hw/cpu.py<br>
@@ -0,0 +1,164 @@<br>
+# SPDX-License-Identifier: BSD-3-Clause<br>
+# Copyright(c) 2022 PANTHEON.tech s.r.o.<br>
+<br>
+import dataclasses<br>
+from abc import ABC, abstractmethod<br>
+from collections.abc import Iterable<br>
+<br>
+from framework.config import CPU, CPUList<br>
+<br>
+<br>
+@dataclasses.dataclass(slots=True, frozen=True)<br>
+class CPUAmount:<br>
+ """<br>
+ Define the amounts of cpus to use. If sockets is not None, socket_amount<br>
+ is ignored.<br>
+ """<br>
+<br>
+ cpus_per_core: int = 1<br>
+ cores_per_socket: int = 2<br>
+ socket_amount: int = 1<br>
+ sockets: list[int] | None = None<br>
+<br>
+<br>
+class CPUFilter(ABC):<br>
+ """<br>
+ Filter according to the input filter specifier. Each filter needs to be<br>
+ implemented in a derived class.<br>
+ This class only implements operations common to all filters, such as sorting<br>
+ the list to be filtered beforehand.<br>
+ """<br>
+<br>
+ _filter_specifier: CPUAmount | CPUList<br>
+ _cpus_to_filter: list[CPU]<br>
+<br>
+ def __init__(<br>
+ self,<br>
+ core_list: list[CPU],<br>
+ filter_specifier: CPUAmount | CPUList,<br>
+ ascending: bool = True,<br>
+ ) -> None:<br>
+ self._filter_specifier = filter_specifier<br>
+<br>
+ # sorting by core is needed in case hyperthreading is enabled<br>
+ self._cpus_to_filter = sorted(<br>
+ core_list, key=lambda x: x.core, reverse=not ascending<br>
+ )<br>
+ self.filter()<br>
+<br>
+ @abstractmethod<br>
+ def filter(self) -> list[CPU]:<br>
+ """<br>
+ Use the input self._filter_specifier to filter self._cpus_to_filter<br>
+ and return the list of filtered CPUs. self._cpus_to_filter is a<br>
+ sorter copy of the original list, so it may be modified.<br>
+ """<br>
+<br>
+<br>
+class CPUAmountFilter(CPUFilter):<br>
+ """<br>
+ Filter the input list of CPUs according to specified rules:<br>
+ Use cores from the specified amount of sockets or from the specified socket ids.<br>
+ If sockets is specified, it takes precedence over socket_amount.<br>
+ From each of those sockets, use only cores_per_socket of cores.<br>
+ And for each core, use cpus_per_core of cpus. Hypertheading<br>
+ must be enabled for this to take effect.<br>
+ If ascending is True, use cores with the lowest numerical id first<br>
+ and continue in ascending order. If False, start with the highest<br>
+ id and continue in descending order. This ordering affects which<br>
+ sockets to consider first as well.<br>
+ """<br>
+<br>
+ _filter_specifier: CPUAmount<br>
+<br>
+ def filter(self) -> list[CPU]:<br>
+ return self._filter_cpus(self._filter_sockets(self._cpus_to_filter))<br>
+<br>
+ def _filter_sockets(self, cpus_to_filter: Iterable[CPU]) -> list[CPU]:<br>
+ allowed_sockets: set[int] = set()<br>
+ socket_amount = self._filter_specifier.socket_amount<br>
+ if self._filter_specifier.sockets:<br>
+ socket_amount = len(self._filter_specifier.sockets)<br>
+ allowed_sockets = set(self._filter_specifier.sockets)<br>
+<br>
+ filtered_cpus = []<br>
+ for cpu in cpus_to_filter:<br>
+ if not self._filter_specifier.sockets:<br>
+ if len(allowed_sockets) < socket_amount:<br>
+ allowed_sockets.add(cpu.socket)<br>
+ if cpu.socket in allowed_sockets:<br>
+ filtered_cpus.append(cpu)<br>
+<br>
+ if len(allowed_sockets) < socket_amount:<br>
+ raise ValueError(<br>
+ f"The amount of sockets from which to use cores "<br>
+ f"({socket_amount}) exceeds the actual amount present "<br>
+ f"on the node ({len(allowed_sockets)})"<br>
+ )<br>
+<br>
+ return filtered_cpus<br>
+<br>
+ def _filter_cpus(self, cpus_to_filter: Iterable[CPU]) -> list[CPU]:<br>
+ # no need to use ordered dict, from Python3.7 the dict<br>
+ # insertion order is preserved (LIFO).<br>
+ allowed_cpu_per_core_count_map: dict[int, int] = {}<br>
+ filtered_cpus = []<br>
+ for cpu in cpus_to_filter:<br>
+ if cpu.core in allowed_cpu_per_core_count_map:<br>
+ cpu_count = allowed_cpu_per_core_count_map[cpu.core]<br>
+ if self._filter_specifier.cpus_per_core > cpu_count:<br>
+ # only add cpus of the given core<br>
+ allowed_cpu_per_core_count_map[cpu.core] += 1<br>
+ filtered_cpus.append(cpu)<br>
+ else:<br>
+ raise ValueError(<br>
+ f"The amount of CPUs per core to use "<br>
+ f"({self._filter_specifier.cpus_per_core}) "<br>
+ f"exceeds the actual amount present. Is hyperthreading enabled?"<br>
+ )<br>
+ elif self._filter_specifier.cores_per_socket > len(<br>
+ allowed_cpu_per_core_count_map<br>
+ ):<br>
+ # only add cpus if we need more<br>
+ allowed_cpu_per_core_count_map[cpu.core] = 1<br>
+ filtered_cpus.append(cpu)<br>
+ else:<br>
+ # cpus are sorted by core, at this point we won't encounter new cores<br>
+ break<br>
+<br>
+ cores_per_socket = len(allowed_cpu_per_core_count_map)<br>
+ if cores_per_socket < self._filter_specifier.cores_per_socket:<br>
+ raise ValueError(<br>
+ f"The amount of cores per socket to use "<br>
+ f"({self._filter_specifier.cores_per_socket}) "<br>
+ f"exceeds the actual amount present ({cores_per_socket})"<br>
+ )<br>
+<br>
+ return filtered_cpus<br>
+<br>
+<br>
+class CPUListFilter(CPUFilter):<br>
+ """<br>
+ Filter the input list of CPUs according to the input list of<br>
+ core indices.<br>
+ An empty CPUList won't filter anything.<br>
+ """<br>
+<br>
+ _filter_specifier: CPUList<br>
+<br>
+ def filter(self) -> list[CPU]:<br>
+ if not len(self._filter_specifier.cpu_list):<br>
+ return self._cpus_to_filter<br>
+<br>
+ filtered_cpus = []<br>
+ for core in self._cpus_to_filter:<br>
+ if core.cpu in self._filter_specifier.cpu_list:<br>
+ filtered_cpus.append(core)<br>
+<br>
+ if len(filtered_cpus) != len(self._filter_specifier.cpu_list):<br>
+ raise ValueError(<br>
+ f"Not all cpus from {self._filter_specifier.cpu_list} were found"<br>
+ f"among {self._cpus_to_filter}"<br>
+ )<br>
+<br>
+ return filtered_cpus<br>
diff --git a/dts/framework/testbed_model/node/node.py b/dts/framework/testbed_model/node/node.py<br>
index 86654e55ae..5ee7023335 100644<br>
--- a/dts/framework/testbed_model/node/node.py<br>
+++ b/dts/framework/testbed_model/node/node.py<br>
@@ -8,13 +8,16 @@<br>
"""<br>
<br>
from framework.config import (<br>
+ CPU,<br>
BuildTargetConfiguration,<br>
+ CPUList,<br>
ExecutionConfiguration,<br>
NodeConfiguration,<br>
)<br>
from framework.exception import NodeCleanupError, NodeSetupError, convert_exception<br>
from framework.logger import DTSLOG, getLogger<br>
from framework.remote_session import OSSession, create_session<br>
+from framework.testbed_model.hw import CPUAmount, cpu_filter<br>
<br>
<br>
class Node(object):<br>
@@ -28,6 +31,7 @@ class Node(object):<br>
main_session: OSSession<br>
logger: DTSLOG<br>
config: NodeConfiguration<br>
+ cpus: list[CPU]<br>
_other_sessions: list[OSSession]<br>
<br>
def __init__(self, node_config: NodeConfiguration):<br>
@@ -38,6 +42,7 @@ def __init__(self, node_config: NodeConfiguration):<br>
self.logger = getLogger(<a href="http://self.name" rel="noreferrer" target="_blank">self.name</a>)<br>
<a href="http://self.logger.info" rel="noreferrer" target="_blank">self.logger.info</a>(f"Created node: {<a href="http://self.name" rel="noreferrer" target="_blank">self.name</a>}")<br>
self.main_session = create_session(self.config, <a href="http://self.name" rel="noreferrer" target="_blank">self.name</a>, self.logger)<br>
+ self._get_remote_cpus()<br>
<br>
@convert_exception(NodeSetupError)<br>
def setup_execution(self, execution_config: ExecutionConfiguration) -> None:<br>
@@ -109,6 +114,37 @@ def create_session(self, name: str) -> OSSession:<br>
self._other_sessions.append(connection)<br>
return connection<br>
<br>
+ def filter_cpus(<br>
+ self,<br>
+ filter_specifier: CPUAmount | CPUList,<br>
+ ascending: bool = True,<br>
+ ) -> list[CPU]:<br>
+ """<br>
+ Filter the logical cpus found on the Node according to specified rules:<br>
+ Use cores from the specified amount of sockets or from the specified<br>
+ socket ids. If sockets is specified, it takes precedence over socket_amount.<br>
+ From each of those sockets, use only cpus_per_socket of cores.<br>
+ And for each core, use cpus_per_core of cpus. Hypertheading<br>
+ must be enabled for this to take effect.<br>
+ If ascending is True, use cores with the lowest numerical id first<br>
+ and continue in ascending order. If False, start with the highest<br>
+ id and continue in descending order. This ordering affects which<br>
+ sockets to consider first as well.<br>
+ """<br>
+ <a href="http://self.logger.info" rel="noreferrer" target="_blank">self.logger.info</a>("Filtering ")<br>
+ return cpu_filter(<br>
+ self.cpus,<br>
+ filter_specifier,<br>
+ ascending,<br>
+ ).filter()<br>
+<br>
+ def _get_remote_cpus(self) -> None:<br>
+ """<br>
+ Scan cpus in the remote OS and store a list of CPUs.<br>
+ """<br>
+ <a href="http://self.logger.info" rel="noreferrer" target="_blank">self.logger.info</a>("Getting CPU information.")<br>
+ self.cpus = self.main_session.get_remote_cpus(self.config.bypass_core0)<br>
+<br>
def close(self) -> None:<br>
"""<br>
Close all connections and free other resources.<br>
diff --git a/dts/framework/testbed_model/node/sut_node.py b/dts/framework/testbed_model/node/sut_node.py<br>
index 53268a7565..ff3be845b4 100644<br>
--- a/dts/framework/testbed_model/node/sut_node.py<br>
+++ b/dts/framework/testbed_model/node/sut_node.py<br>
@@ -4,10 +4,13 @@<br>
<br>
import os<br>
import tarfile<br>
+import time<br>
from pathlib import PurePath<br>
<br>
-from framework.config import BuildTargetConfiguration, NodeConfiguration<br>
+from framework.config import CPU, BuildTargetConfiguration, CPUList, NodeConfiguration<br>
+from framework.remote_session import OSSession<br>
from framework.settings import SETTINGS<br>
+from framework.testbed_model.hw import CPUAmount, CPUListFilter<br>
from framework.utils import EnvVarsDict, skip_setup<br>
<br>
from .node import Node<br>
@@ -21,19 +24,31 @@ class SutNode(Node):<br>
Another key capability is building DPDK according to given build target.<br>
"""<br>
<br>
+ cpus: list[CPU]<br>
+ dpdk_prefix_list: list[str]<br>
+ dpdk_prefix_subfix: str<br>
_build_target_config: BuildTargetConfiguration | None<br>
_env_vars: EnvVarsDict<br>
_remote_tmp_dir: PurePath<br>
__remote_dpdk_dir: PurePath | None<br>
_app_compile_timeout: float<br>
+ _dpdk_kill_session: OSSession | None<br>
<br>
def __init__(self, node_config: NodeConfiguration):<br>
super(SutNode, self).__init__(node_config)<br>
+ self.dpdk_prefix_list = []<br>
self._build_target_config = None<br>
self._env_vars = EnvVarsDict()<br>
self._remote_tmp_dir = self.main_session.get_remote_tmp_dir()<br>
self.__remote_dpdk_dir = None<br>
self._app_compile_timeout = 90<br>
+ self._dpdk_kill_session = None<br>
+<br>
+ # filter the node cpus according to user config<br>
+ self.cpus = CPUListFilter(self.cpus, self.config.cpus).filter()<br>
+ self.dpdk_prefix_subfix = (<br>
+ f"{str(os.getpid())}_{time.strftime('%Y%m%d%H%M%S', time.localtime())}"<br>
+ )<br>
<br>
@property<br>
def _remote_dpdk_dir(self) -> PurePath:<br>
@@ -142,3 +157,164 @@ def build_dpdk_app(self, app_name: str) -> PurePath:<br>
return self.main_session.join_remote_path(<br>
build_dir, "examples", f"dpdk-{app_name}"<br>
)<br>
+<br>
+ def kill_cleanup_dpdk_apps(self) -> None:<br>
+ """<br>
+ Kill all dpdk applications on the SUT. Cleanup hugepages.<br>
+ """<br>
+ if self._dpdk_kill_session and self._dpdk_kill_session.is_alive():<br>
+ # we can use the session if it exists and responds<br>
+ self._dpdk_kill_session.kill_cleanup_dpdk_apps(self.dpdk_prefix_list)<br>
+ else:<br>
+ # otherwise, we need to (re)create it<br>
+ self._dpdk_kill_session = self.create_session("dpdk_kill")<br>
+ self.dpdk_prefix_list = []<br>
+<br>
+ def create_eal_parameters(<br>
+ self,<br>
+ fixed_prefix: bool = False,<br>
+ core_filter_specifier: CPUAmount | CPUList = CPUAmount(),<br>
+ ascending_cores: bool = True,<br>
+ prefix: str = "",<br>
+ no_pci: bool = False,<br>
+ vdevs: list[str] = None,<br></blockquote><div><br></div><div>I would prefer to have vdevs be a list of objects, even if for now that class just takes a string in its constructor. Later on we can add subclasses for specific vdevs that might see heavy use, such as librte_net_pcap and crypto_openssl. </div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+ other_eal_param: str = "",<br>
+ ) -> str:<br>
+ """<br>
+ Generate eal parameters character string;<br>
+ :param fixed_prefix: use fixed file-prefix or not, when it is true,<br>
+ the file-prefix will not be added a timestamp<br>
+ :param core_filter_specifier: an amount of cpus/cores/sockets to use<br>
+ or a list of cpu ids to use.<br>
+ The default will select one cpu for each of two cores<br>
+ on one socket, in ascending order of core ids.<br>
+ :param ascending_cores: True, use cores with the lowest numerical id first<br>
+ and continue in ascending order. If False, start with the<br>
+ highest id and continue in descending order. This ordering<br>
+ affects which sockets to consider first as well.<br>
+ :param prefix: set file prefix string, eg:<br>
+ prefix='vf';<br>
+ :param no_pci: switch of disable PCI bus eg:<br>
+ no_pci=True;<br>
+ :param vdevs: virtual device list, eg:<br>
+ vdevs=['net_ring0', 'net_ring1'];<br>
+ :param other_eal_param: user defined DPDK eal parameters, eg:<br>
+ other_eal_param='--single-file-segments';<br>
+ :return: eal param string, eg:<br>
+ '-c 0xf -a 0000:88:00.0 --file-prefix=dpdk_1112_20190809143420';<br>
+ if DPDK version < 20.11-rc4, eal_str eg:<br>
+ '-c 0xf -w 0000:88:00.0 --file-prefix=dpdk_1112_20190809143420';<br>
+ """<br>
+ if vdevs is None:<br>
+ vdevs = []<br>
+<br>
+ config = {<br>
+ "core_filter_specifier": core_filter_specifier,<br>
+ "ascending_cores": ascending_cores,<br>
+ "prefix": prefix,<br>
+ "no_pci": no_pci,<br>
+ "vdevs": vdevs,<br>
+ "other_eal_param": other_eal_param,<br>
+ }<br>
+<br>
+ eal_parameter_creator = _EalParameter(<br>
+ sut_node=self, fixed_prefix=fixed_prefix, **config<br>
+ )<br>
+ eal_str = eal_parameter_creator.make_eal_param()<br>
+<br>
+ return eal_str<br>
+<br>
+<br>
+class _EalParameter(object):<br>
+ def __init__(<br>
+ self,<br>
+ sut_node: SutNode,<br>
+ fixed_prefix: bool,<br>
+ core_filter_specifier: CPUAmount | CPUList,<br>
+ ascending_cores: bool,<br>
+ prefix: str,<br>
+ no_pci: bool,<br>
+ vdevs: list[str],<br>
+ other_eal_param: str,<br>
+ ):<br>
+ """<br>
+ Generate eal parameters character string;<br>
+ :param sut_node: SUT Node;<br>
+ :param fixed_prefix: use fixed file-prefix or not, when it is true,<br>
+ he file-prefix will not be added a timestamp<br>
+ :param core_filter_specifier: an amount of cpus/cores/sockets to use<br>
+ or a list of cpu ids to use.<br>
+ :param ascending_cores: True, use cores with the lowest numerical id first<br>
+ and continue in ascending order. If False, start with the<br>
+ highest id and continue in descending order. This ordering<br>
+ affects which sockets to consider first as well.<br>
+ :param prefix: set file prefix string, eg:<br>
+ prefix='vf';<br>
+ :param no_pci: switch of disable PCI bus eg:<br>
+ no_pci=True;<br>
+ :param vdevs: virtual device list, eg:<br>
+ vdevs=['net_ring0', 'net_ring1'];<br>
+ :param other_eal_param: user defined DPDK eal parameters, eg:<br>
+ other_eal_param='--single-file-segments';<br>
+ """<br>
+ self.os = sut_node.config.os<br>
+ self.fixed_prefix = fixed_prefix<br>
+ self.sut_node = sut_node<br>
+ self.core_filter_specifier = core_filter_specifier<br>
+ self.ascending_cores = ascending_cores<br>
+ self.prefix = prefix<br>
+ self.no_pci = no_pci<br>
+ self.vdevs = vdevs<br>
+ self.other_eal_param = other_eal_param<br>
+<br>
+ def _make_lcores_param(self) -> str:<br>
+ filtered_cpus = self.sut_node.filter_cpus(<br>
+ self.core_filter_specifier, self.ascending_cores<br>
+ )<br>
+ return f"-l {CPUList(filtered_cpus)}"<br>
+<br>
+ def _make_memory_channels(self) -> str:<br>
+ param_template = "-n {}"<br>
+ return param_template.format(self.sut_node.config.memory_channels)<br>
+<br>
+ def _make_no_pci_param(self) -> str:<br>
+ if self.no_pci is True:<br>
+ return "--no-pci"<br>
+ else:<br>
+ return ""<br>
+<br>
+ def _make_prefix_param(self) -> str:<br>
+ if self.prefix == "":<br>
+ fixed_file_prefix = f"dpdk_{self.sut_node.dpdk_prefix_subfix}"<br>
+ else:<br>
+ fixed_file_prefix = self.prefix<br>
+ if not self.fixed_prefix:<br>
+ fixed_file_prefix = (<br>
+ f"{fixed_file_prefix}_{self.sut_node.dpdk_prefix_subfix}"<br>
+ )<br>
+ fixed_file_prefix = self._do_os_handle_with_prefix_param(fixed_file_prefix)<br>
+ return fixed_file_prefix<br>
+<br>
+ def _make_vdevs_param(self) -> str:<br>
+ if len(self.vdevs) == 0:<br>
+ return ""<br>
+ else:<br>
+ return " ".join(f"--vdev {vdev}" for vdev in self.vdevs)<br>
+<br>
+ def _do_os_handle_with_prefix_param(self, file_prefix: str) -> str:<br>
+ self.sut_node.dpdk_prefix_list.append(file_prefix)<br>
+ return f"--file-prefix={file_prefix}"<br>
+<br>
+ def make_eal_param(self) -> str:<br>
+ _eal_str = " ".join(<br>
+ [<br>
+ self._make_lcores_param(),<br>
+ self._make_memory_channels(),<br>
+ self._make_prefix_param(),<br>
+ self._make_no_pci_param(),<br>
+ self._make_vdevs_param(),<br>
+ # append user defined eal parameters<br>
+ self.other_eal_param,<br>
+ ]<br>
+ )<br>
+ return _eal_str<br>
diff --git a/dts/framework/utils.py b/dts/framework/utils.py<br>
index 91e58f3218..3c2f0adff9 100644<br>
--- a/dts/framework/utils.py<br>
+++ b/dts/framework/utils.py<br>
@@ -32,6 +32,26 @@ def skip_setup(func) -> Callable[..., None]:<br>
return func<br>
<br>
<br>
+def expand_range(range_str: str) -> list[int]:<br>
+ """<br>
+ Process range string into a list of integers. There are two possible formats:<br>
+ n - a single integer<br>
+ n-m - a range of integers<br>
+<br>
+ The returned range includes both n and m. Empty string returns an empty list.<br>
+ """<br>
+ expanded_range: list[int] = []<br>
+ if range_str:<br>
+ range_boundaries = range_str.split("-")<br>
+ # will throw an exception when items in range_boundaries can't be converted,<br>
+ # serving as type check<br>
+ expanded_range.extend(<br>
+ range(int(range_boundaries[0]), int(range_boundaries[-1]) + 1)<br>
+ )<br>
+<br>
+ return expanded_range<br>
+<br>
+<br>
def GREEN(text: str) -> str:<br>
return f"\u001B[32;1m{str(text)}\u001B[0m"<br>
<br>
-- <br>
2.30.2<br>
<br>
</blockquote></div></div>