[dts] [next] [PATCH 1/3] framework: support argument: --update-expected.

[Liu, Yong]

Lijuan,
This feature request is more like suite level requirement. In current implementation, all suite configurations will be overwrite if this flag is enabled. Thus may affect other suites.

How about give the permission to suite? Suite can update its own configuration when update option is enabled. It can limit the impact.

Thanks,
Marvin

> -----Original Message-----
> From: dts [mailto:dts-bounces at dpdk.org] On Behalf Of Lijuan Tu
> Sent: Friday, July 20, 2018 11:22 PM
> To: dts at dpdk.org; pmacarth at iol.unh.edu; alialnu at mellanox.com
> Cc: Tu, Lijuan <lijuan.tu at intel.com>
> Subject: [dts] [next] [PATCH 1/3] framework: support argument: --update-
> expected.
> 
> A DPDK Performance Test Lab are established, and Specification requires
> DTS to support an --update-expected argument which will cause the
> script to update values based on the results of the current test run.
> 
> Signed-off-by: Lijuan Tu <lijuan.tu at intel.com>
> ---
>  framework/config.py    | 11 +++++++++++
>  framework/dts.py       |  6 +++++-
>  framework/main.py      |  6 +++++-
>  framework/settings.py  |  1 +
>  framework/test_case.py | 22 ++++++++++++++++++----
>  5 files changed, 40 insertions(+), 6 deletions(-)
> 
> diff --git a/framework/config.py b/framework/config.py
> index 628fc6d..0b112d8 100644
> --- a/framework/config.py
> +++ b/framework/config.py
> @@ -147,6 +147,17 @@ class SuiteConf(UserConf):
> 
>          return case_cfg
> 
> +    def update_case_config(self, case_name=""):
> +        """
> +        update section (case_name) of the configure file
> +        """
> +        update_suite_cfg_obj = UserConf(self.config_file)
> +        update_suite_cfg = update_suite_cfg_obj.load_section(case_name)
> +        for key in update_suite_cfg_obj.conf.options(case_name):
> +            update_suite_cfg_obj.conf.set(
> +                case_name, key, str(self.suite_cfg[key]))
> +        update_suite_cfg_obj.conf.write(open(self.config_file, 'w'))
> +
> 
>  class VirtConf(UserConf):
> 
> diff --git a/framework/dts.py b/framework/dts.py
> index 0b2240c..4435418 100644
> --- a/framework/dts.py
> +++ b/framework/dts.py
> @@ -439,7 +439,7 @@ def dts_run_suite(duts, tester, test_suites, target):
>  def run_all(config_file, pkgName, git, patch, skip_setup,
>              read_cache, project, suite_dir, test_cases,
>              base_dir, output_dir, verbose, virttype, debug,
> -            debugcase, re_run, commands):
> +            debugcase, re_run, commands, update_expected):
>      """
>      Main process of DTS, it will run all test suites in the config file.
>      """
> @@ -479,6 +479,10 @@ def run_all(config_file, pkgName, git, patch,
> skip_setup,
>      if debugcase is True:
>          settings.save_global_setting(settings.DEBUG_CASE_SETTING, 'yes')
> 
> +    # enable update-expected
> +    if update_expected is True:
> +        settings.save_global_setting(settings.UPDATE_EXPECTED, 'yes')
> +
>      # init log_handler handler
>      if verbose is True:
>          logger.set_verbose()
> diff --git a/framework/main.py b/framework/main.py
> index 0aa54fd..9d7ef31 100755
> --- a/framework/main.py
> +++ b/framework/main.py
> @@ -143,6 +143,10 @@ parser.add_argument('--commands',
>                      help='run command on tester or dut. The command
> format is ' +
>                      '[commands]:dut|tester:pre-init|post-
> init:check|ignore')
> 
> +parser.add_argument('--update-expected',
> +                    action='store_true',
> +                    help='update expected values based on test results')
> +
>  args = parser.parse_args()
> 
> 
> @@ -159,4 +163,4 @@ dts.run_all(args.config_file, args.snapshot, args.git,
>              args.patch, args.skip_setup, args.read_cache,
>              args.project, args.suite_dir, args.test_cases,
>              args.dir, args.output, args.verbose,args.virttype,
> -            args.debug, args.debugcase, args.re_run, args.commands)
> +            args.debug, args.debugcase, args.re_run, args.commands,
> args.update_expected)
> diff --git a/framework/settings.py b/framework/settings.py
> index 07c3ac6..2561ddb 100644
> --- a/framework/settings.py
> +++ b/framework/settings.py
> @@ -218,6 +218,7 @@ DPDK_RXMODE_SETTING = "DTS_DPDK_RXMODE"
>  DTS_ERROR_ENV = "DTS_RUNNING_ERROR"
>  DTS_CFG_FOLDER = "DTS_CFG_FOLDER"
>  DTS_PARALLEL_SETTING = "DTS_PARALLEL_ENABLE"
> +UPDATE_EXPECTED = "DTS_UPDATE_EXPECTED_ENABLE"
> 
> 
>  """
> diff --git a/framework/test_case.py b/framework/test_case.py
> index a84e2bb..ffa03db 100644
> --- a/framework/test_case.py
> +++ b/framework/test_case.py
> @@ -40,7 +40,9 @@ import time
> 
>  from exception import VerifyFailure, TimeoutException
>  from settings import DRIVERS, NICS, get_nic_name, load_global_setting
> -from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING,
> DEBUG_CASE_SETTING, HOST_DRIVER_SETTING
> +from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING
> +from settings import DEBUG_CASE_SETTING, HOST_DRIVER_SETTING
> +from settings import UPDATE_EXPECTED, SUITE_SECTION_NAME
>  from rst import RstReport
>  from test_result import ResultTable, Result
>  from logger import getLogger
> @@ -254,10 +256,7 @@ class TestCase(object):
>          self._rst_obj.write_title("Test Case: " + case_name)
> 
>          # load suite configuration file here for rerun command
> -        self._suite_conf = SuiteConf(self.suite_name)
> -        self._suite_cfg = self._suite_conf.suite_cfg
>          self._case_cfg = self._suite_conf.load_case_config(case_name)
> -        del(self._suite_conf)
> 

Lijuan,
Code here has mentioned that suite config file will reload for later usage. I think it won't affect your updated expected function.
You should keep it for functionality.

>          case_result = True
>          if self._check_inst is not None:
> @@ -315,6 +314,9 @@ class TestCase(object):
>              self._suite_result.test_case_failed(trace)
>              self.logger.error('Test Case %s Result ERROR: ' % (case_name)
> + trace)
>          finally:
> +            # update expected
> +            if load_global_setting(UPDATE_EXPECTED) == "yes":
> +                self._suite_conf.update_case_config(SUITE_SECTION_NAME)
>              self.tear_down()
>              return case_result
> 
> @@ -375,6 +377,18 @@ class TestCase(object):
>          """
>          return self._suite_cfg
> 
> +    def update_suite_cfg(self, suite_cfg):
> +        """
> +        Update suite based configuration
> +        """
> +        self._suite_cfg = suite_cfg
> +
> +    def update_suite_cfg_ele(self, key, value):
> +        """
> +        update one element of suite configuration
> +        """
> +        self._suite_cfg[key]=value
> +

These functions were not called in test suite nic_single_core_perf. Please make sure they are useful. 

>      def execute_tear_downall(self):
>          """
>          execute suite tear_down_all function
> --
> 1.8.3.1