[dpdk-ci] [PATCH V2] patch-tagging: Added tool to convert tags into dts execution file

[Owen Hilyard]

Please ignore this patch. I messed up directories and this patch was not
created from the correct file.

On Thu, Jan 14, 2021 at 12:59 PM <ohilyard at iol.unh.edu> wrote:

> Fixed parsing bugs with the tag parsing script
>         Parsing the patch file directly proved to be too unreliabler, so
> the approach to patch parsing was changed so that it relies instead on
> using a directory with those patches already applied in them and then use a
> git diff command to get the changed files. This avoids needing to parse the
> patch files directly and should be more reliable.
>
> Added a tool to create an execution file from tags using another execution
> file as a template
>         This tool is intended to allow a list of patch tags to be used in
> combination with an execution file containing information on crbs to only
> change which test suites are run. This tool enables using the mappings in
> the tests_for_tag.cfg file to run only necessary tests. At present, the
> mappings represent how tests are currently run at the UNH-IOL Community
> Lab, with the same tests being run for every patch, with the requested
> change from bug 511 where documentation changes do not cause any testing.
>
> Signed-off-by: Owen Hilyard <ohilyard at iol.unh.edu>
> ---
>  config/tests_for_tag.cfg | 19 ++++++++++++--
>  tools/patch_parser.py    | 54 ++++++++++++++++++++++++----------------
>  2 files changed, 49 insertions(+), 24 deletions(-)
>
> diff --git a/config/tests_for_tag.cfg b/config/tests_for_tag.cfg
> index 77f797f..7d95c4a 100644
> --- a/config/tests_for_tag.cfg
> +++ b/config/tests_for_tag.cfg
> @@ -5,11 +5,26 @@ core = dynamic_config,
>         mtu_update,
>         scatter,
>         stats_checks,
> +       unit_tests_mbuf
> +driver = dynamic_config,
> +       link_status_interrupt,
> +       mac_filter,
> +       mtu_update,
> +       scatter,
> +       stats_checks,
> +       unit_tests_mbuf
> +application = dynamic_config,
> +       link_status_interrupt,
> +       mac_filter,
> +       mtu_update,
> +       scatter,
>         stats_checks,
>         unit_tests_mbuf
> -
>  ; Nothing in documentation
>  documentation =
>
>  [performance]
> -core = nic_single_core_perf,
> +core = nic_single_core_perf
> +driver = nic_single_core_perf
> +application = nic_single_core_perf
> +documentation =
> diff --git a/tools/patch_parser.py b/tools/patch_parser.py
> index 80b8194..73df91f 100755
> --- a/tools/patch_parser.py
> +++ b/tools/patch_parser.py
> @@ -31,30 +31,39 @@
>  # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>  # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>  # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> -
> +import subprocess
>
>  import itertools
>  from configparser import ConfigParser
>  from typing import List, Dict, Set
>  import argparse
> +import re
>
>
>  def get_patch_files(patch_file: str) -> List[str]:
> +    file_match_pattern = re.compile(r"[\w\-\/.]+ +\| +\d+ [+\-]+")
>      with open(patch_file, 'r') as f:
> -        lines = list(itertools.takewhile(
> -            lambda line: line.strip().endswith('+') or
> line.strip().endswith('-'),
> -            itertools.dropwhile(
> -                lambda line: not line.strip().startswith("---"),
> -                f.readlines()
> +        file_lines = f.readlines()
> +        lines = list(filter(
> +            lambda line: file_match_pattern.match(line.strip()),
> +            itertools.takewhile(
> +                lambda line: not line.startswith('---'),
> +                list(
> +                    itertools.dropwhile(
> +                        lambda line: not line.strip().startswith("---"),
> +                        file_lines
> +                    )
> +                )[1:]
>              )
>          ))
>          filenames = map(lambda line: line.strip().split(' ')[0], lines)
> -        # takewhile includes the --- which starts the filenames
> -        return list(filenames)[1:]
> +        return list(filenames)
>
>
>  def get_all_files_from_patches(patch_files: List[str]) -> Set[str]:
> -    return set(itertools.chain.from_iterable(map(get_patch_files,
> patch_files)))
> +    num_patch_files: int = len(patch_files)
> +    files = subprocess.check_output(f"cd dpdk && git diff --name-only
> HEAD~{num_patch_files}", shell=True).decode('utf-8').splitlines()
> +    return set(files)
>
>
>  def parse_comma_delimited_list_from_string(mod_str: str) -> List[str]:
> @@ -80,21 +89,22 @@ def get_tags_for_patches(patch_files: Set[str],
> dir_attrs: Dict[str, Set[str]])
>      ))
>
>
> -parser = argparse.ArgumentParser(
> -    description='Takes a patch file and a config file and creates a list
> of tags for that patch')
> -parser.add_argument('config_file_path', help='The path to
> patch_parser.cfg', default='config/patch_parser.cfg')
> -parser.add_argument('patch_file_paths', help='A list of patch files',
> type=str, metavar='patch file', nargs='+')
> +if __name__ == '__main__':
> +    parser = argparse.ArgumentParser(
> +        description='Takes a patch file and a config file and creates a
> list of tags for that patch')
> +    parser.add_argument('config_file_path', help='The path to
> patch_parser.cfg', default='config/patch_parser.cfg')
> +    parser.add_argument('patch_file_paths', help='A list of patch files',
> type=str, metavar='patch file', nargs='+')
>
> -args = parser.parse_args()
> +    args = parser.parse_args()
>
> -conf_obj = ConfigParser()
> -conf_obj.read(args.config_file_path)
> +    conf_obj = ConfigParser()
> +    conf_obj.read(args.config_file_path)
>
> -patch_files = get_all_files_from_patches(args.patch_file_paths)
> -dir_attrs = get_dictionary_attributes_from_config_file(conf_obj)
> -priority_list =
> parse_comma_delimited_list_from_string(conf_obj['Priority']['priority_list'])
> +    patch_files = get_all_files_from_patches(args.patch_file_paths)
> +    dir_attrs = get_dictionary_attributes_from_config_file(conf_obj)
> +    priority_list =
> parse_comma_delimited_list_from_string(conf_obj['Priority']['priority_list'])
>
> -unordered_tags: Set[str] = get_tags_for_patches(patch_files, dir_attrs)
> -ordered_tags: List[str] = [tag for tag in priority_list if tag in
> unordered_tags]
> +    unordered_tags: Set[str] = get_tags_for_patches(patch_files,
> dir_attrs)
> +    ordered_tags: List[str] = [tag for tag in priority_list if tag in
> unordered_tags]
>
> -print("\n".join(ordered_tags))
> +    print("\n".join(ordered_tags))
> --
> 2.27.0
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mails.dpdk.org/archives/ci/attachments/20210114/5a4351bc/attachment-0001.htm>