Small refact on FSM's testcase finish logic

Cleaned up /test_coordinator py files

ioppytest/test_coordinator/amqp_connector.py

@@ -18,7 +18,6 @@ from messages import *
 # TODO get filter from config of the TEDs
 COAP_CLIENT_IUT_MODE = 'user-assisted'
 COAP_SERVER_IUT_MODE = 'automated'
-ANALYSIS_MODE = 'post_mortem'  # either step_by_step or post_mortem
 
 # if left empty => packet_sniffer chooses the loopback
 # TODO send flag to sniffer telling him to look for a tun interface instead!

ioppytest/test_coordinator/coordinator.py

@@ -73,7 +73,7 @@ class Coordinator(CoordinatorAmqpInterface):
         else:
             self.event = event
 
-    def generate_testcases_verdict(self, received_event):
+    def save_current_testcase_report_to_file(self, received_event):
         verdict_info = {}
         info1 = self.testsuite.get_current_testcase().to_dict(verbose=True)
         info2 = self.testsuite.get_testcase_report()
@@ -252,10 +252,10 @@ class Coordinator(CoordinatorAmqpInterface):
 
     def finish_testcase(self):
         """
-        :return:
         """
+
         if self.testsuite.check_testcase_finished() is False:
-            msg = 'expected testcase to be finished'
+            msg = 'Expected testcase to be finished.'
             logger.error(msg)
             ls_tc, ls_steps = self.testsuite.get_detailed_status()
 
@@ -263,142 +263,140 @@ class Coordinator(CoordinatorAmqpInterface):
             logger.error('steps: %s' % ls_steps)
             raise CoordinatorError(msg)
 
+        # Get 'pointer' to current test case and update status
         current_tc = self.testsuite.get_current_testcase()
         current_tc.change_state('analyzing')
         current_tc.current_step = None
 
-        # get TC params
+        # Get TC params
         tc_id = current_tc.id
         tc_ref = current_tc.uri
 
-        # Finish sniffer and get PCAP
+        # Stop sniffing packets for this test case
         logger.debug("Sending sniffer stop request...")
         self.call_service_sniffer_stop()
-        time.sleep(0.5)
+        time.sleep(0.1)
+
+        # Get capture of test case
+        logger.debug("Sending get capture request to sniffer...")
+        sniffer_response = self.call_service_sniffer_get_capture(capture_id=tc_id)
+
+        # Save .pcap file locally
+        try:
+            if sniffer_response.ok:
+                pcap_file_base64 = sniffer_response.value
+                filename = sniffer_response.filename
 
-        # TODO break this function in smaller pieces
-        if ANALYSIS_MODE == 'post_mortem':
+                with open(os.path.join(PCAP_DIR, filename), "wb") as pcap_file:
+                    nb = pcap_file.write(base64.b64decode(pcap_file_base64))
+                    logger.debug("Pcap correctly saved (%d Bytes) at %s" % (nb, TMPDIR))
+            else:
+                error_msg = 'Error encountered with packet sniffer: %s' % repr(sniffer_response)
+                logger.warning(error_msg)
 
-            tat_response = None
-            gen_verdict = ''
-            gen_description = ''
-            report = []
-            error_msg = ''
+                # generate error verdict/report
+                current_tc.generate_testcase_report(err_msg=error_msg)
+                self.save_current_testcase_report_to_file(None)
 
-            logger.debug("Sending get capture request to sniffer...")
-            sniffer_response = self.call_service_sniffer_get_capture(capture_id=tc_id)
+                # change tc state
+                current_tc.change_state('finished')
+                return
 
-            # let's try to save the file and then push it to results repo
-            try:
-                if sniffer_response.ok:
-                    pcap_file_base64 = sniffer_response.value
-                    filename = sniffer_response.filename
-
-                    # save PCAP to file
-                    with open(os.path.join(PCAP_DIR, filename), "wb") as pcap_file:
-                        nb = pcap_file.write(base64.b64decode(pcap_file_base64))
-                        logger.debug("Pcap correctly saved (%d Bytes) at %s" % (nb, TMPDIR))
-
-                    logger.debug("Sending PCAP file to TAT for analysis...")
-
-                    # Forwards PCAP to TAT to get CHECKs results
-                    try:
-                        tat_response = self.call_service_testcase_analysis(
-                            protocol=self.testsuite_name,
-                            testcase_id=tc_id,
-                            testcase_ref=tc_ref,
-                            file_enc="pcap_base64",
-                            filename=tc_id + ".pcap",
-                            value=pcap_file_base64)
-
-                    except AmqpSynchCallTimeoutError as e:
-                        error_msg += "TAT didnt answer to the analysis request"
-                        logger.error(error_msg)
-
-                    if tat_response and tat_response.ok:
-
-                        logger.info("Response received from TAT: %s " % repr(tat_response))
-                        # Save the json object received
-                        json_file = os.path.join(
-                            TMPDIR,
-                            '%s_analysis.json' % tc_id
-                        )
-
-                        with open(json_file, 'w') as f:
-                            f.write(tat_response.to_json())
-
-                        # let's process the partial verdicts from TAT's answer
-                        # format : [[partial verdict : str, description : str]]
-                        partial_verd = []
-                        step_count = 0
-                        ls_len = len(tat_response.partial_verdicts)
-                        for item in tat_response.partial_verdicts:
-                            # let's partial verdict id
-                            step_count += 1
-                            p = ("frame_check_[{}/{}]".format(step_count, ls_len), item[0], item[1])
-                            partial_verd.append(p)
-                            logger.debug("Processing partial verdict received from TAT: %s" % str(p))
-
-                        # generates a general verdict considering other steps partial verdicts besides TAT's
-                        gen_verdict, gen_description, report = current_tc.generate_testcases_verdict(partial_verd)
-
-                    else:
-                        error_msg += 'Error message: %s (err.code: %s)' % (tat_response.error_message,
-                                                                           tat_response.error_code)
-                        logger.warning(error_msg)
-
-                        # generate verdict and verdict description
-                        try:
-                            gen_description = error_msg
-                            gen_verdict = 'inconclusive'
-                        except AttributeError:
-                            gen_description = error_msg
-                            gen_verdict = 'error'
-
-                        report = []
-                else:
-                    error_msg += 'Error encountered with packet sniffer: %s' % repr(sniffer_response)
-                    logger.warning(error_msg)
-                    gen_verdict = 'error'
-                    gen_description = error_msg
-                    report = []
-
-            except AttributeError as ae:
-                error_msg += 'Failed to process Sniffer response. Wrongly formated response? : %s' % repr(
-                    sniffer_response)
-                logger.error(error_msg)
-                gen_verdict = 'error'
-                gen_description = error_msg
-                report = []
-
-            # TODO this should be hanlded directly by generate_testcases_verdict method
-            # save sent message in RESULTS dir
-            final_report = OrderedDict()
-            final_report['verdict'] = gen_verdict
-            final_report['description'] = gen_description
-            final_report['partial_verdicts'] = report
-
-            # TODO this should be hanlded directly by generate_testcases_verdict method
-            # lets generate test case report
-            current_tc.report = final_report
-
-            # Save the final verdict as json
-            json_file = os.path.join(
-                TMPDIR,
-                tc_id + '_verdict.json'
-            )
-            with open(json_file, 'w') as f:
-                json.dump(final_report, f)
+        except AttributeError as ae:
+            error_msg = 'Failed to process packet sniffer response. ' \
+                        'Wrongly formatted response? : %s' % repr(sniffer_response)
+            logger.error(error_msg)
+
+            # generate error verdict/report
+            current_tc.generate_testcase_report(err_msg=error_msg)
+            self.save_current_testcase_report_to_file(None)
 
             # change tc state
             current_tc.change_state('finished')
-            logger.info("General verdict generated: %s" % json.dumps(current_tc.report))
+            return
 
-        else:
-            # TODO implement step-by-step analysis
-            raise NotImplementedError()
+        # Forward .pcap to TAT to get CHECKs results
+        try:
+            tat_response = self.call_service_testcase_analysis(
+                protocol=self.testsuite_name,
+                testcase_id=tc_id,
+                testcase_ref=tc_ref,
+                file_enc="pcap_base64",
+                filename=tc_id + ".pcap",
+                value=pcap_file_base64)
+
+        except AmqpSynchCallTimeoutError as e:
+            error_msg = "TAT service timeout error"
+            logger.error(error_msg)
+
+            # generate error verdict/report
+            current_tc.generate_testcase_report(err_msg=error_msg)
+            self.save_current_testcase_report_to_file(None)
+
+            # change tc state
+            current_tc.change_state('finished')
+            return
+
+        # Process TAT response and create final test case report
+        try:
+            if tat_response.ok:
+
+                logger.info("Response received from TAT: %s " % repr(tat_response))
+                # Save the json object received
+                json_file = os.path.join(
+                    TMPDIR,
+                    '%s_analysis.json' % tc_id
+                )
+
+                with open(json_file, 'w') as f:
+                    f.write(tat_response.to_json())
+
+                # let's process the partial verdicts from TAT's answer
+                # format : [[partial verdict : str, description : str]]
+                partial_verd = []
+                step_count = 0
+                ls_len = len(tat_response.partial_verdicts)
+                for item in tat_response.partial_verdicts:
+                    # let's partial verdict id
+                    step_count += 1
+                    p = ("frame_check_[{}/{}]".format(step_count, ls_len), item[0], item[1])
+                    partial_verd.append(p)
+                    logger.debug("Processing partial verdict received from TAT: %s" % str(p))
+
+            else:
+                error_msg = 'Error response from TAT : %s (err.code: %s)' % (tat_response.error_message,
+                                                                             tat_response.error_code)
+                logger.warning(error_msg)
+
+                # generate error verdict/report
+                current_tc.generate_testcase_report(err_msg=error_msg)
+                self.save_current_testcase_report_to_file(None)
+
+                # change tc state
+                current_tc.change_state('finished')
+                return
+
+        except AttributeError:
+            error_msg = "TAT didn't send a correctly formatted message, we got %s"% repr(tat_response)
+            logger.error(error_msg)
+
+            # generate error verdict/report
+            current_tc.generate_testcase_report(err_msg=error_msg)
+            self.save_current_testcase_report_to_file(None)
+
+            # change tc state
+            current_tc.change_state('finished')
+            return
+
+        # Create final test case report from TAT partial verdicts
+        # Note: report is generated from TAT partial verdicts (CHECKS), as well as IUTs' VERIFY steps
+        current_tc.generate_testcase_report(partial_verd)
+        logger.info("General verdict generated: %s" % json.dumps(current_tc.report))
+        self.save_current_testcase_report_to_file(None)
 
-        return current_tc.report
+        # change tc state
+        current_tc.change_state('finished')
+        self.save_current_testcase_report_to_file(None)
 
     def handle_testcase_restart(self, received_event):
         """

ioppytest/test_coordinator/states_and_transitions.py

@@ -59,7 +59,7 @@ states = [
         'name': 'testcase_finished',
         'on_enter': [
             'notify_testcase_finished',
-            'generate_testcases_verdict',
+            'save_current_testcase_report_to_file',
             'notify_testcase_verdict',
             'to_preparing_next_testcase'],  # jumps to following state, this makes testcase_finished a transition state
         'on_exit': []

ioppytest/test_suite/testsuite.py

@@ -118,8 +118,6 @@ COMPONENT_ID = 'testsuite'
 logger = logging.getLogger(COMPONENT_ID)
 logger.setLevel(LOG_LEVEL)
 
-ANALYSIS_MODE = 'post_mortem'  # either step_by_step or post_mortem
-
 # AMQP log handler with f-interop's json formatter
 rabbitmq_handler = RabbitMQHandler(AMQP_URL, COMPONENT_ID)
 json_formatter = JsonFormatter()
@@ -320,13 +318,7 @@ class TestSuite:
             report_item = {'testcase_id': tc.id}
             if tc.report is None:
                 logger.warning("Empty report found. Generating dummy report for skipped testcase : %s" % tc.id)
-                # TODO this should be hanlded directly by generate_testcases_verdict method
-                gen_verdict, gen_description, rep = tc.generate_testcases_verdict(None)
-                final_report = OrderedDict()
-                final_report['verdict'] = gen_verdict
-                final_report['description'] = gen_description
-                final_report['partial_verdicts'] = rep
-                tc.report = final_report
+                tc.generate_testcase_report(err_msg="No test case verdict/report found for test case")
 
             report_item.update(tc.report)
             report.append(report_item)
@@ -662,13 +654,10 @@ class TestSuite:
 
         if testcase_id:
             assert type(testcase_id) is str
-        # assigns the one which is not None:
+
         tc = self.get_testcase(testcase_id) or self.get_current_testcase()
 
-        if tc:
-            return tc.report
-        else:
-            return None
+        return tc.report if tc else None
 
     def reinit_testcase(self, testcase_id):
         """
@@ -883,11 +872,16 @@ class TestConfig:
 
 class Step():
     def __init__(self, step_id, type, description, node=None):
-        self.id = step_id
         assert type in ("stimuli", "check", "verify", "feature")
+        self.id = step_id
         self.type = type
+        self.state = None
         self.description = description
 
+        # stimuli steps dont have a verdict, they are IUT actions
+        if type != 'stimuli':
+            self.partial_verdict = Verdict()
+
         # stimuli and verify step MUST have a iut field in the YAML file
         if type == 'stimuli' or type == 'verify':
             assert node is not None
@@ -895,11 +889,6 @@ class Step():
         else:
             self.iut = None
 
-        # stimulis step dont get partial verdict
-        self.partial_verdict = Verdict()
-
-        self.state = None
-
     def __repr__(self):
         node = ''
         mode = ''
@@ -911,16 +900,16 @@ class Step():
 
     def reinit(self):
 
-        if self.type in ('check', 'verify', 'feature'):
+        logger.debug('Step (re)initing for: step_id: %s, step_type: %s' % (self.id, self.type))
+
+        if self.type in ('check', 'feature'):
             self.partial_verdict = Verdict()
+            self.change_state('postponed')
+
+        elif self.type == 'verify':
+            self.partial_verdict = Verdict()
+            self.change_state(None)
 
-            # when using post_mortem analysis mode all checks are postponed , and analysis is done at the end of the TC
-            logger.debug('Processing step init, step_id: %s, step_type: %s, ANALYSIS_MODE is %s' % (
-                self.id, self.type, ANALYSIS_MODE))
-            if self.type == 'check' or self.type == 'feature' and ANALYSIS_MODE == 'post_mortem':
-                self.change_state('postponed')
-            else:
-                self.change_state(None)
         else:  # its a stimuli
             self.change_state(None)
 
@@ -931,7 +920,7 @@ class Step():
             step_dict['step_type'] = self.type
             step_dict['step_info'] = self.description
             step_dict['step_state'] = self.state
-            # it the step is a stimuli then lets add the IUT info(note that checks dont have that info)
+            # let's return IUT info for steps associated to an IUT
             if self.type == 'stimuli' or self.type == 'verify':
                 step_dict.update(self.iut.to_dict())
         return step_dict
@@ -1078,21 +1067,29 @@ class TestCase:
             logger.debug("[TESTCASE] - all steps in TC are either finished (or pending).")
             return True
 
-    def generate_testcases_verdict(self, tat_post_mortem_analysis_report=None):
+    def generate_testcase_report(self, tat_analysis_report=None, err_msg=None):
         """
-        Generates the final verdict of TC and report taking into account the CHECKs and VERIFYs of the testcase
-        :return: tuple: (final_verdict, verdict_description, tc_report) ,
-                 where final_verdict in ("None", "error", "inconclusive", "pass" , "fail")
-                 where description is String type
-                 where tc report is a list :
-                                [(step, step_partial_verdict, step_verdict_info, associated_frame_id (can be null))]
+        Generates the final verdict (report) of TC taking into consideration the CHECKs and VERIFYs of the testcase.
+        Updates report attribute from tc object
+
+        :return:
+                tc report format: [(step, step_partial_verdict, step_verdict_info, associated_frame_id (can be null))]
         """
+
+        if err_msg:
+            self.report = {
+                'verdict': 'error',
+                'description': err_msg,
+                'partial_verdicts': [],
+            }
+
+            return self.report
+
         logger.debug("[VERDICT GENERATION] starting the verdict generation")
 
         if self.state is None or self.state == 'skipped' or self.state == 'aborted':
             return ('None', 'Testcase %s was %s.' % (self.id, self.state), [])
 
-        # TODO hanlde frame id associated to the step , used for GUI purposes
         if self.check_all_steps_finished() is False:
             logger.warning("Found non finished steps: %s" % json.dumps(self.seq_to_dict(verbose=False)))
             return
@@ -1104,24 +1101,26 @@ class TestCase:
             # for the verdict we use the info in the checks and verify steps
             if step.type in ("check", "verify", "feature"):
 
-                logger.debug("[VERDICT GENERATION] Processing step %s" % step.id)
-
                 if step.state == "postponed":
-                    tc_report.append((step.id, None, "%s step: postponed" % step.type.upper(), ""))
+                    #tc_report.append((step.id, None, "%s step: postponed" % step.type.upper(), ""))
+                    pass
+
                 elif step.state == "finished":
                     tc_report.append(
-                        (step.id, step.partial_verdict.get_value(), step.partial_verdict.get_message(), ""))
+                        (step.id, step.partial_verdict.get_value(), step.partial_verdict.get_message(), "")
+                    )
                     # update global verdict
                     final_verdict.update(step.partial_verdict.get_value(), step.partial_verdict.get_message())
+
                 else:
                     msg = "step %s not ready for analysis" % (step.id)
                     logger.error("[VERDICT GENERATION] " + msg)
                     raise TestSuiteError(msg)
 
         # append at the end of the report the analysis done a posteriori (if any)
-        if tat_post_mortem_analysis_report and len(tat_post_mortem_analysis_report) != 0:
-            logger.info('Processing TAT partial verdict: ' + str(tat_post_mortem_analysis_report))
-            for item in tat_post_mortem_analysis_report:
+        if tat_analysis_report and len(tat_analysis_report) != 0:
+            logger.info('Processing TAT partial verdict: ' + str(tat_analysis_report))
+            for item in tat_analysis_report:
                 # TODO process the items correctly
                 tc_report.append(item)
                 final_verdict.update(item[1], item[2])
@@ -1133,12 +1132,16 @@ class TestCase:
         # hack to overwrite the final verdict MESSAGE in case of pass
         if final_verdict.get_value() == 'pass':
             final_verdict.update('pass', 'No interoperability error was detected.')
-            logger.debug("[VERDICT GENERATION] Test case executed correctly, a PASS was issued.")
         else:
-            logger.debug("[VERDICT GENERATION] Test case executed correctly, but FAIL was issued as verdict.")
             logger.debug("[VERDICT GENERATION] info: %s' " % final_verdict.get_value())
 
-        return final_verdict.get_value(), final_verdict.get_message(), tc_report
+        self.report = {
+            'verdict': final_verdict.get_value(),
+            'description': final_verdict.get_message(),
+            'partial_verdicts': tc_report,
+        }
+
+        return self.report
 
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@@ -1170,7 +1173,7 @@ def testcase_constructor(loader, node):
     instance = TestCase.__new__(TestCase)
     yield instance
     state = loader.construct_mapping(node, deep=True)
-    logging.debug("pasing test case: " + str(state))
+    #logging.debug("pasing test case: " + str(state))
     instance.__init__(**state)