Commit 723e027b authored by Federico Sismondi's avatar Federico Sismondi
Browse files

Small refact on FSM's testcase finish logic

Cleaned up /test_coordinator py files
parents 18d2b0eb f915e273
Pipeline #3144 canceled with stage
in 0 seconds
......@@ -18,7 +18,6 @@ from messages import *
# TODO get filter from config of the TEDs
COAP_CLIENT_IUT_MODE = 'user-assisted'
COAP_SERVER_IUT_MODE = 'automated'
ANALYSIS_MODE = 'post_mortem' # either step_by_step or post_mortem
# if left empty => packet_sniffer chooses the loopback
# TODO send flag to sniffer telling him to look for a tun interface instead!
......
......@@ -73,7 +73,7 @@ class Coordinator(CoordinatorAmqpInterface):
else:
self.event = event
def generate_testcases_verdict(self, received_event):
def save_current_testcase_report_to_file(self, received_event):
verdict_info = {}
info1 = self.testsuite.get_current_testcase().to_dict(verbose=True)
info2 = self.testsuite.get_testcase_report()
......@@ -252,10 +252,10 @@ class Coordinator(CoordinatorAmqpInterface):
def finish_testcase(self):
"""
:return:
"""
if self.testsuite.check_testcase_finished() is False:
msg = 'expected testcase to be finished'
msg = 'Expected testcase to be finished.'
logger.error(msg)
ls_tc, ls_steps = self.testsuite.get_detailed_status()
......@@ -263,142 +263,140 @@ class Coordinator(CoordinatorAmqpInterface):
logger.error('steps: %s' % ls_steps)
raise CoordinatorError(msg)
# Get 'pointer' to current test case and update status
current_tc = self.testsuite.get_current_testcase()
current_tc.change_state('analyzing')
current_tc.current_step = None
# get TC params
# Get TC params
tc_id = current_tc.id
tc_ref = current_tc.uri
# Finish sniffer and get PCAP
# Stop sniffing packets for this test case
logger.debug("Sending sniffer stop request...")
self.call_service_sniffer_stop()
time.sleep(0.5)
time.sleep(0.1)
# Get capture of test case
logger.debug("Sending get capture request to sniffer...")
sniffer_response = self.call_service_sniffer_get_capture(capture_id=tc_id)
# Save .pcap file locally
try:
if sniffer_response.ok:
pcap_file_base64 = sniffer_response.value
filename = sniffer_response.filename
# TODO break this function in smaller pieces
if ANALYSIS_MODE == 'post_mortem':
with open(os.path.join(PCAP_DIR, filename), "wb") as pcap_file:
nb = pcap_file.write(base64.b64decode(pcap_file_base64))
logger.debug("Pcap correctly saved (%d Bytes) at %s" % (nb, TMPDIR))
else:
error_msg = 'Error encountered with packet sniffer: %s' % repr(sniffer_response)
logger.warning(error_msg)
tat_response = None
gen_verdict = ''
gen_description = ''
report = []
error_msg = ''
# generate error verdict/report
current_tc.generate_testcase_report(err_msg=error_msg)
self.save_current_testcase_report_to_file(None)
logger.debug("Sending get capture request to sniffer...")
sniffer_response = self.call_service_sniffer_get_capture(capture_id=tc_id)
# change tc state
current_tc.change_state('finished')
return
# let's try to save the file and then push it to results repo
try:
if sniffer_response.ok:
pcap_file_base64 = sniffer_response.value
filename = sniffer_response.filename
# save PCAP to file
with open(os.path.join(PCAP_DIR, filename), "wb") as pcap_file:
nb = pcap_file.write(base64.b64decode(pcap_file_base64))
logger.debug("Pcap correctly saved (%d Bytes) at %s" % (nb, TMPDIR))
logger.debug("Sending PCAP file to TAT for analysis...")
# Forwards PCAP to TAT to get CHECKs results
try:
tat_response = self.call_service_testcase_analysis(
protocol=self.testsuite_name,
testcase_id=tc_id,
testcase_ref=tc_ref,
file_enc="pcap_base64",
filename=tc_id + ".pcap",
value=pcap_file_base64)
except AmqpSynchCallTimeoutError as e:
error_msg += "TAT didnt answer to the analysis request"
logger.error(error_msg)
if tat_response and tat_response.ok:
logger.info("Response received from TAT: %s " % repr(tat_response))
# Save the json object received
json_file = os.path.join(
TMPDIR,
'%s_analysis.json' % tc_id
)
with open(json_file, 'w') as f:
f.write(tat_response.to_json())
# let's process the partial verdicts from TAT's answer
# format : [[partial verdict : str, description : str]]
partial_verd = []
step_count = 0
ls_len = len(tat_response.partial_verdicts)
for item in tat_response.partial_verdicts:
# let's partial verdict id
step_count += 1
p = ("frame_check_[{}/{}]".format(step_count, ls_len), item[0], item[1])
partial_verd.append(p)
logger.debug("Processing partial verdict received from TAT: %s" % str(p))
# generates a general verdict considering other steps partial verdicts besides TAT's
gen_verdict, gen_description, report = current_tc.generate_testcases_verdict(partial_verd)
else:
error_msg += 'Error message: %s (err.code: %s)' % (tat_response.error_message,
tat_response.error_code)
logger.warning(error_msg)
# generate verdict and verdict description
try:
gen_description = error_msg
gen_verdict = 'inconclusive'
except AttributeError:
gen_description = error_msg
gen_verdict = 'error'
report = []
else:
error_msg += 'Error encountered with packet sniffer: %s' % repr(sniffer_response)
logger.warning(error_msg)
gen_verdict = 'error'
gen_description = error_msg
report = []
except AttributeError as ae:
error_msg += 'Failed to process Sniffer response. Wrongly formated response? : %s' % repr(
sniffer_response)
logger.error(error_msg)
gen_verdict = 'error'
gen_description = error_msg
report = []
# TODO this should be hanlded directly by generate_testcases_verdict method
# save sent message in RESULTS dir
final_report = OrderedDict()
final_report['verdict'] = gen_verdict
final_report['description'] = gen_description
final_report['partial_verdicts'] = report
# TODO this should be hanlded directly by generate_testcases_verdict method
# lets generate test case report
current_tc.report = final_report
# Save the final verdict as json
json_file = os.path.join(
TMPDIR,
tc_id + '_verdict.json'
)
with open(json_file, 'w') as f:
json.dump(final_report, f)
except AttributeError as ae:
error_msg = 'Failed to process packet sniffer response. ' \
'Wrongly formatted response? : %s' % repr(sniffer_response)
logger.error(error_msg)
# generate error verdict/report
current_tc.generate_testcase_report(err_msg=error_msg)
self.save_current_testcase_report_to_file(None)
# change tc state
current_tc.change_state('finished')
logger.info("General verdict generated: %s" % json.dumps(current_tc.report))
return
else:
# TODO implement step-by-step analysis
raise NotImplementedError()
# Forward .pcap to TAT to get CHECKs results
try:
tat_response = self.call_service_testcase_analysis(
protocol=self.testsuite_name,
testcase_id=tc_id,
testcase_ref=tc_ref,
file_enc="pcap_base64",
filename=tc_id + ".pcap",
value=pcap_file_base64)
except AmqpSynchCallTimeoutError as e:
error_msg = "TAT service timeout error"
logger.error(error_msg)
# generate error verdict/report
current_tc.generate_testcase_report(err_msg=error_msg)
self.save_current_testcase_report_to_file(None)
# change tc state
current_tc.change_state('finished')
return
# Process TAT response and create final test case report
try:
if tat_response.ok:
logger.info("Response received from TAT: %s " % repr(tat_response))
# Save the json object received
json_file = os.path.join(
TMPDIR,
'%s_analysis.json' % tc_id
)
with open(json_file, 'w') as f:
f.write(tat_response.to_json())
# let's process the partial verdicts from TAT's answer
# format : [[partial verdict : str, description : str]]
partial_verd = []
step_count = 0
ls_len = len(tat_response.partial_verdicts)
for item in tat_response.partial_verdicts:
# let's partial verdict id
step_count += 1
p = ("frame_check_[{}/{}]".format(step_count, ls_len), item[0], item[1])
partial_verd.append(p)
logger.debug("Processing partial verdict received from TAT: %s" % str(p))
else:
error_msg = 'Error response from TAT : %s (err.code: %s)' % (tat_response.error_message,
tat_response.error_code)
logger.warning(error_msg)
# generate error verdict/report
current_tc.generate_testcase_report(err_msg=error_msg)
self.save_current_testcase_report_to_file(None)
# change tc state
current_tc.change_state('finished')
return
except AttributeError:
error_msg = "TAT didn't send a correctly formatted message, we got %s"% repr(tat_response)
logger.error(error_msg)
# generate error verdict/report
current_tc.generate_testcase_report(err_msg=error_msg)
self.save_current_testcase_report_to_file(None)
# change tc state
current_tc.change_state('finished')
return
# Create final test case report from TAT partial verdicts
# Note: report is generated from TAT partial verdicts (CHECKS), as well as IUTs' VERIFY steps
current_tc.generate_testcase_report(partial_verd)
logger.info("General verdict generated: %s" % json.dumps(current_tc.report))
self.save_current_testcase_report_to_file(None)
return current_tc.report
# change tc state
current_tc.change_state('finished')
self.save_current_testcase_report_to_file(None)
def handle_testcase_restart(self, received_event):
"""
......
......@@ -59,7 +59,7 @@ states = [
'name': 'testcase_finished',
'on_enter': [
'notify_testcase_finished',
'generate_testcases_verdict',
'save_current_testcase_report_to_file',
'notify_testcase_verdict',
'to_preparing_next_testcase'], # jumps to following state, this makes testcase_finished a transition state
'on_exit': []
......
......@@ -118,8 +118,6 @@ COMPONENT_ID = 'testsuite'
logger = logging.getLogger(COMPONENT_ID)
logger.setLevel(LOG_LEVEL)
ANALYSIS_MODE = 'post_mortem' # either step_by_step or post_mortem
# AMQP log handler with f-interop's json formatter
rabbitmq_handler = RabbitMQHandler(AMQP_URL, COMPONENT_ID)
json_formatter = JsonFormatter()
......@@ -320,13 +318,7 @@ class TestSuite:
report_item = {'testcase_id': tc.id}
if tc.report is None:
logger.warning("Empty report found. Generating dummy report for skipped testcase : %s" % tc.id)
# TODO this should be hanlded directly by generate_testcases_verdict method
gen_verdict, gen_description, rep = tc.generate_testcases_verdict(None)
final_report = OrderedDict()
final_report['verdict'] = gen_verdict
final_report['description'] = gen_description
final_report['partial_verdicts'] = rep
tc.report = final_report
tc.generate_testcase_report(err_msg="No test case verdict/report found for test case")
report_item.update(tc.report)
report.append(report_item)
......@@ -662,13 +654,10 @@ class TestSuite:
if testcase_id:
assert type(testcase_id) is str
# assigns the one which is not None:
tc = self.get_testcase(testcase_id) or self.get_current_testcase()
if tc:
return tc.report
else:
return None
return tc.report if tc else None
def reinit_testcase(self, testcase_id):
"""
......@@ -883,11 +872,16 @@ class TestConfig:
class Step():
def __init__(self, step_id, type, description, node=None):
self.id = step_id
assert type in ("stimuli", "check", "verify", "feature")
self.id = step_id
self.type = type
self.state = None
self.description = description
# stimuli steps dont have a verdict, they are IUT actions
if type != 'stimuli':
self.partial_verdict = Verdict()
# stimuli and verify step MUST have a iut field in the YAML file
if type == 'stimuli' or type == 'verify':
assert node is not None
......@@ -895,11 +889,6 @@ class Step():
else:
self.iut = None
# stimulis step dont get partial verdict
self.partial_verdict = Verdict()
self.state = None
def __repr__(self):
node = ''
mode = ''
......@@ -911,16 +900,16 @@ class Step():
def reinit(self):
if self.type in ('check', 'verify', 'feature'):
logger.debug('Step (re)initing for: step_id: %s, step_type: %s' % (self.id, self.type))
if self.type in ('check', 'feature'):
self.partial_verdict = Verdict()
self.change_state('postponed')
elif self.type == 'verify':
self.partial_verdict = Verdict()
self.change_state(None)
# when using post_mortem analysis mode all checks are postponed , and analysis is done at the end of the TC
logger.debug('Processing step init, step_id: %s, step_type: %s, ANALYSIS_MODE is %s' % (
self.id, self.type, ANALYSIS_MODE))
if self.type == 'check' or self.type == 'feature' and ANALYSIS_MODE == 'post_mortem':
self.change_state('postponed')
else:
self.change_state(None)
else: # its a stimuli
self.change_state(None)
......@@ -931,7 +920,7 @@ class Step():
step_dict['step_type'] = self.type
step_dict['step_info'] = self.description
step_dict['step_state'] = self.state
# it the step is a stimuli then lets add the IUT info(note that checks dont have that info)
# let's return IUT info for steps associated to an IUT
if self.type == 'stimuli' or self.type == 'verify':
step_dict.update(self.iut.to_dict())
return step_dict
......@@ -1078,21 +1067,29 @@ class TestCase:
logger.debug("[TESTCASE] - all steps in TC are either finished (or pending).")
return True
def generate_testcases_verdict(self, tat_post_mortem_analysis_report=None):
def generate_testcase_report(self, tat_analysis_report=None, err_msg=None):
"""
Generates the final verdict of TC and report taking into account the CHECKs and VERIFYs of the testcase
:return: tuple: (final_verdict, verdict_description, tc_report) ,
where final_verdict in ("None", "error", "inconclusive", "pass" , "fail")
where description is String type
where tc report is a list :
[(step, step_partial_verdict, step_verdict_info, associated_frame_id (can be null))]
Generates the final verdict (report) of TC taking into consideration the CHECKs and VERIFYs of the testcase.
Updates report attribute from tc object
:return:
tc report format: [(step, step_partial_verdict, step_verdict_info, associated_frame_id (can be null))]
"""
if err_msg:
self.report = {
'verdict': 'error',
'description': err_msg,
'partial_verdicts': [],
}
return self.report
logger.debug("[VERDICT GENERATION] starting the verdict generation")
if self.state is None or self.state == 'skipped' or self.state == 'aborted':
return ('None', 'Testcase %s was %s.' % (self.id, self.state), [])
# TODO hanlde frame id associated to the step , used for GUI purposes
if self.check_all_steps_finished() is False:
logger.warning("Found non finished steps: %s" % json.dumps(self.seq_to_dict(verbose=False)))
return
......@@ -1104,24 +1101,26 @@ class TestCase:
# for the verdict we use the info in the checks and verify steps
if step.type in ("check", "verify", "feature"):
logger.debug("[VERDICT GENERATION] Processing step %s" % step.id)
if step.state == "postponed":
tc_report.append((step.id, None, "%s step: postponed" % step.type.upper(), ""))
#tc_report.append((step.id, None, "%s step: postponed" % step.type.upper(), ""))
pass
elif step.state == "finished":
tc_report.append(
(step.id, step.partial_verdict.get_value(), step.partial_verdict.get_message(), ""))
(step.id, step.partial_verdict.get_value(), step.partial_verdict.get_message(), "")
)
# update global verdict
final_verdict.update(step.partial_verdict.get_value(), step.partial_verdict.get_message())
else:
msg = "step %s not ready for analysis" % (step.id)
logger.error("[VERDICT GENERATION] " + msg)
raise TestSuiteError(msg)
# append at the end of the report the analysis done a posteriori (if any)
if tat_post_mortem_analysis_report and len(tat_post_mortem_analysis_report) != 0:
logger.info('Processing TAT partial verdict: ' + str(tat_post_mortem_analysis_report))
for item in tat_post_mortem_analysis_report:
if tat_analysis_report and len(tat_analysis_report) != 0:
logger.info('Processing TAT partial verdict: ' + str(tat_analysis_report))
for item in tat_analysis_report:
# TODO process the items correctly
tc_report.append(item)
final_verdict.update(item[1], item[2])
......@@ -1133,12 +1132,16 @@ class TestCase:
# hack to overwrite the final verdict MESSAGE in case of pass
if final_verdict.get_value() == 'pass':
final_verdict.update('pass', 'No interoperability error was detected.')
logger.debug("[VERDICT GENERATION] Test case executed correctly, a PASS was issued.")
else:
logger.debug("[VERDICT GENERATION] Test case executed correctly, but FAIL was issued as verdict.")
logger.debug("[VERDICT GENERATION] info: %s' " % final_verdict.get_value())
return final_verdict.get_value(), final_verdict.get_message(), tc_report
self.report = {
'verdict': final_verdict.get_value(),
'description': final_verdict.get_message(),
'partial_verdicts': tc_report,
}
return self.report
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
......@@ -1170,7 +1173,7 @@ def testcase_constructor(loader, node):
instance = TestCase.__new__(TestCase)
yield instance
state = loader.construct_mapping(node, deep=True)
logging.debug("pasing test case: " + str(state))
#logging.debug("pasing test case: " + str(state))
instance.__init__(**state)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment