[dpdk-ci] [PATCH V2] patch-tagging: Added tool to convert tags into dts execution file

[Aaron Conole]

Owen Hilyard <ohilyard at iol.unh.edu> writes:

> Please ignore this patch. I messed up directories and this patch was not created from the correct file. 

Is there an update for this that I missed?

> On Thu, Jan 14, 2021 at 12:59 PM <ohilyard at iol.unh.edu> wrote:
>
>  Fixed parsing bugs with the tag parsing script
>          Parsing the patch file directly proved to be too unreliabler, so the approach to patch parsing was changed so that
>  it relies instead on using a directory with those patches already applied in them and then use a git diff command to get
>  the changed files. This avoids needing to parse the patch files directly and should be more reliable.
>
>  Added a tool to create an execution file from tags using another execution file as a template
>          This tool is intended to allow a list of patch tags to be used in combination with an execution file containing
>  information on crbs to only change which test suites are run. This tool enables using the mappings in the
>  tests_for_tag.cfg file to run only necessary tests. At present, the mappings represent how tests are currently run at the
>  UNH-IOL Community Lab, with the same tests being run for every patch, with the requested change from bug 511
>  where documentation changes do not cause any testing.
>
>  Signed-off-by: Owen Hilyard <ohilyard at iol.unh.edu>
>  ---
>   config/tests_for_tag.cfg | 19 ++++++++++++--
>   tools/patch_parser.py    | 54 ++++++++++++++++++++++++----------------
>   2 files changed, 49 insertions(+), 24 deletions(-)
>
>  diff --git a/config/tests_for_tag.cfg b/config/tests_for_tag.cfg
>  index 77f797f..7d95c4a 100644
>  --- a/config/tests_for_tag.cfg
>  +++ b/config/tests_for_tag.cfg
>  @@ -5,11 +5,26 @@ core = dynamic_config,
>          mtu_update,
>          scatter,
>          stats_checks,
>  +       unit_tests_mbuf
>  +driver = dynamic_config,
>  +       link_status_interrupt,
>  +       mac_filter,
>  +       mtu_update,
>  +       scatter,
>  +       stats_checks,
>  +       unit_tests_mbuf
>  +application = dynamic_config,
>  +       link_status_interrupt,
>  +       mac_filter,
>  +       mtu_update,
>  +       scatter,
>          stats_checks,
>          unit_tests_mbuf
>  -
>   ; Nothing in documentation
>   documentation =
>
>   [performance]
>  -core = nic_single_core_perf,
>  +core = nic_single_core_perf
>  +driver = nic_single_core_perf
>  +application = nic_single_core_perf
>  +documentation =
>  diff --git a/tools/patch_parser.py b/tools/patch_parser.py
>  index 80b8194..73df91f 100755
>  --- a/tools/patch_parser.py
>  +++ b/tools/patch_parser.py
>  @@ -31,30 +31,39 @@
>   # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>   # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>   # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>  -
>  +import subprocess
>
>   import itertools
>   from configparser import ConfigParser
>   from typing import List, Dict, Set
>   import argparse
>  +import re
>
>   def get_patch_files(patch_file: str) -> List[str]:
>  +    file_match_pattern = re.compile(r"[\w\-\/.]+ +\| +\d+ [+\-]+")
>       with open(patch_file, 'r') as f:
>  -        lines = list(itertools.takewhile(
>  -            lambda line: line.strip().endswith('+') or line.strip().endswith('-'),
>  -            itertools.dropwhile(
>  -                lambda line: not line.strip().startswith("---"),
>  -                f.readlines()
>  +        file_lines = f.readlines()
>  +        lines = list(filter(
>  +            lambda line: file_match_pattern.match(line.strip()),
>  +            itertools.takewhile(
>  +                lambda line: not line.startswith('---'),
>  +                list(
>  +                    itertools.dropwhile(
>  +                        lambda line: not line.strip().startswith("---"),
>  +                        file_lines
>  +                    )
>  +                )[1:]
>               )
>           ))
>           filenames = map(lambda line: line.strip().split(' ')[0], lines)
>  -        # takewhile includes the --- which starts the filenames
>  -        return list(filenames)[1:]
>  +        return list(filenames)
>
>   def get_all_files_from_patches(patch_files: List[str]) -> Set[str]:
>  -    return set(itertools.chain.from_iterable(map(get_patch_files, patch_files)))
>  +    num_patch_files: int = len(patch_files)
>  +    files = subprocess.check_output(f"cd dpdk && git diff --name-only HEAD~{num_patch_files}", shell=True).decode
>  ('utf-8').splitlines()
>  +    return set(files)
>
>   def parse_comma_delimited_list_from_string(mod_str: str) -> List[str]:
>  @@ -80,21 +89,22 @@ def get_tags_for_patches(patch_files: Set[str], dir_attrs: Dict[str, Set[str]])
>       ))
>
>  -parser = argparse.ArgumentParser(
>  -    description='Takes a patch file and a config file and creates a list of tags for that patch')
>  -parser.add_argument('config_file_path', help='The path to patch_parser.cfg', default='config/patch_parser.cfg')
>  -parser.add_argument('patch_file_paths', help='A list of patch files', type=str, metavar='patch file', nargs='+')
>  +if __name__ == '__main__':
>  +    parser = argparse.ArgumentParser(
>  +        description='Takes a patch file and a config file and creates a list of tags for that patch')
>  +    parser.add_argument('config_file_path', help='The path to patch_parser.cfg', default='config/patch_parser.cfg')
>  +    parser.add_argument('patch_file_paths', help='A list of patch files', type=str, metavar='patch file', nargs='+')
>
>  -args = parser.parse_args()
>  +    args = parser.parse_args()
>
>  -conf_obj = ConfigParser()
>  -conf_obj.read(args.config_file_path)
>  +    conf_obj = ConfigParser()
>  +    conf_obj.read(args.config_file_path)
>
>  -patch_files = get_all_files_from_patches(args.patch_file_paths)
>  -dir_attrs = get_dictionary_attributes_from_config_file(conf_obj)
>  -priority_list = parse_comma_delimited_list_from_string(conf_obj['Priority']['priority_list'])
>  +    patch_files = get_all_files_from_patches(args.patch_file_paths)
>  +    dir_attrs = get_dictionary_attributes_from_config_file(conf_obj)
>  +    priority_list = parse_comma_delimited_list_from_string(conf_obj['Priority']['priority_list'])
>
>  -unordered_tags: Set[str] = get_tags_for_patches(patch_files, dir_attrs)
>  -ordered_tags: List[str] = [tag for tag in priority_list if tag in unordered_tags]
>  +    unordered_tags: Set[str] = get_tags_for_patches(patch_files, dir_attrs)
>  +    ordered_tags: List[str] = [tag for tag in priority_list if tag in unordered_tags]
>
>  -print("\n".join(ordered_tags))
>  +    print("\n".join(ordered_tags))
>  -- 
>  2.27.0