<div dir="ltr">Acked-by: Jeremy Spweock <<a href="mailto:jspweock@iol.unh.edu" target="_blank">jspweock@iol.unh.edu</a>> </div> <div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Thu, Apr 20, 2023 at 10:16 AM Juraj Linkeš <juraj.linkes@pantheon.tech> wrote: </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">Add additional convenience options for specifying what DPDK version to test. Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech> --- dts/framework/config/__init__.py | 11 +-- dts/framework/settings.py | 20 ++--- dts/framework/utils.py | 140 +++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 19 deletions(-) diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index ebb0823ff5..a4b73483e6 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -11,21 +11,14 @@ import os.path import pathlib from dataclasses import dataclass -from enum import Enum, auto, unique +from enum import auto, unique from typing import Any, TypedDict import warlock # type: ignore import yaml from framework.settings import SETTINGS - - -class StrEnum(Enum): - @staticmethod - def _generate_next_value_( - name: str, start: int, count: int, last_values: object - ) -> str: - return name +from framework.utils import StrEnum @unique diff --git a/dts/framework/settings.py b/dts/framework/settings.py index 71955f4581..cfa39d011b 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Any, TypeVar -from .exception import ConfigurationError +from .utils import DPDKGitTarball _T = TypeVar("_T") @@ -124,11 +124,13 @@ def _get_parser() -> argparse.ArgumentParser: parser.add_argument( "--tarball", "--snapshot", + "--git-ref", action=_env_arg("DTS_DPDK_TARBALL"), default="dpdk.tar.xz", type=Path, - help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball " - "which will be used in testing.", + help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball or a git commit ID, " + "tag ID or tree ID to test. To test local changes, first commit them, " + "then use the commit ID with this option.", ) parser.add_argument( @@ -160,21 +162,19 @@ def _get_parser() -> argparse.ArgumentParser: return parser -def _check_tarball_path(parsed_args: argparse.Namespace) -> None: - if not os.path.exists(parsed_args.tarball): - raise ConfigurationError(f"DPDK tarball '{parsed_args.tarball}' doesn't exist.") - - def _get_settings() -> _Settings: parsed_args = _get_parser().parse_args() - _check_tarball_path(parsed_args) return _Settings( config_file_path=parsed_args.config_file, output_dir=parsed_args.output_dir, timeout=parsed_args.timeout, verbose=(parsed_args.verbose == "Y"), skip_setup=(parsed_args.skip_setup == "Y"), - dpdk_tarball_path=parsed_args.tarball, + dpdk_tarball_path=Path( + DPDKGitTarball(parsed_args.tarball, parsed_args.output_dir) + ) + if not os.path.exists(parsed_args.tarball) + else Path(parsed_args.tarball), compile_timeout=parsed_args.compile_timeout, test_cases=parsed_args.test_cases.split(",") if parsed_args.test_cases else [], re_run=parsed_args.re_run, diff --git a/dts/framework/utils.py b/dts/framework/utils.py index 55e0b0ef0e..0623106b78 100644 --- a/dts/framework/utils.py +++ b/dts/framework/utils.py @@ -3,7 +3,26 @@ # Copyright(c) 2022-2023 PANTHEON.tech s.r.o. # Copyright(c) 2022-2023 University of New Hampshire +import atexit +import os +import subprocess import sys +from enum import Enum +from pathlib import Path +from subprocess import SubprocessError + +from .exception import ConfigurationError + + +class StrEnum(Enum): + @staticmethod + def _generate_next_value_( + name: str, start: int, count: int, last_values: object + ) -> str: + return name + + def __str__(self) -> str: + return <a href="http://self.name" rel="noreferrer" target="_blank">self.name</a> def check_dts_python_version() -> None: @@ -80,3 +99,124 @@ def __init__(self, default_library: str | None = None, **dpdk_args: str | bool): def __str__(self) -> str: return " ".join(f"{self._default_library} {self._dpdk_args}".split()) + + +class _TarCompressionFormat(StrEnum): + """Compression formats that tar can use. + + Enum names are the shell compression commands + and Enum values are the associated file extensions. + """ + + gzip = "gz" + compress = "Z" + bzip2 = "bz2" + lzip = "lz" + lzma = "lzma" + lzop = "lzo" + xz = "xz" + zstd = "zst" + + +class DPDKGitTarball(object): + """Create a compressed tarball of DPDK from the repository. + + The DPDK version is specified with git object git_ref. + The tarball will be compressed with _TarCompressionFormat, + which must be supported by the DTS execution environment. + The resulting tarball will be put into output_dir. + + The class supports the os.PathLike protocol, + which is used to get the Path of the tarball:: + + from pathlib import Path + tarball = DPDKGitTarball("HEAD", "output") + tarball_path = Path(tarball) + + Arguments: + git_ref: A git commit ID, tag ID or tree ID. + output_dir: The directory where to put the resulting tarball. + tar_compression_format: The compression format to use. + """ + + _git_ref: str + _tar_compression_format: _TarCompressionFormat + _tarball_dir: Path + _tarball_name: str + _tarball_path: Path | None + + def __init__( + self, + git_ref: str, + output_dir: str, + tar_compression_format: _TarCompressionFormat = _TarCompressionFormat.xz, + ): + self._git_ref = git_ref + self._tar_compression_format = tar_compression_format + + self._tarball_dir = Path(output_dir, "tarball") + + self._get_commit_id() + self._create_tarball_dir() + + self._tarball_name = ( + f"dpdk-tarball-{self._git_ref}.tar.{self._tar_compression_format.value}" + ) + self._tarball_path = self._check_tarball_path() + if not self._tarball_path: + self._create_tarball() + + def _get_commit_id(self) -> None: + result = subprocess.run( + ["git", "rev-parse", "--verify", self._git_ref], + text=True, + capture_output=True, + ) + if result.returncode != 0: + raise ConfigurationError( + f"{self._git_ref} is neither a path to an existing DPDK " + "archive nor a valid git reference.\n" + f"Command: {result.args}\n" + f"Stdout: {result.stdout}\n" + f"Stderr: {result.stderr}" + ) + self._git_ref = result.stdout.strip() + + def _create_tarball_dir(self) -> None: + os.makedirs(self._tarball_dir, exist_ok=True) + + def _check_tarball_path(self) -> Path | None: + if self._tarball_name in os.listdir(self._tarball_dir): + return Path(self._tarball_dir, self._tarball_name) + return None + + def _create_tarball(self) -> None: + self._tarball_path = Path(self._tarball_dir, self._tarball_name) + + atexit.register(self._delete_tarball) + + result = subprocess.run( + 'git -C "$(git rev-parse --show-toplevel)" archive ' + f'{self._git_ref} --prefix="dpdk-tarball-{self._git_ref + os.sep}" | ' + f"{self._tar_compression_format} > {Path(self._tarball_path.absolute())}", + shell=True, + text=True, + capture_output=True, + ) + + if result.returncode != 0: + raise SubprocessError( + f"Git archive creation failed with exit code {result.returncode}.\n" + f"Command: {result.args}\n" + f"Stdout: {result.stdout}\n" + f"Stderr: {result.stderr}" + ) + + atexit.unregister(self._delete_tarball) + + def _delete_tarball(self) -> None: + if self._tarball_path and os.path.exists(self._tarball_path): + os.remove(self._tarball_path) + + def __fspath__(self): + return str(self._tarball_path) -- 2.30.2 </blockquote></div>